





"""simple docstring"""






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A	     =					logging.get_logger(__name__)
A	     =					{'''vocab_file''': '''spiece.model'''}

A	     =					{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

A	     =					{
    '''albert-base-v1''': 512,
    '''albert-large-v1''': 512,
    '''albert-xlarge-v1''': 512,
    '''albert-xxlarge-v1''': 512,
    '''albert-base-v2''': 512,
    '''albert-large-v2''': 512,
    '''albert-xlarge-v2''': 512,
    '''albert-xxlarge-v2''': 512,
}

A	     =					'''▁'''




class 			__lowercase     (     _UpperCamelCase							):







					'''simple docstring'''




					__lowerCAmelCase					=					VOCAB_FILES_NAMES
					__lowerCAmelCase					=					PRETRAINED_VOCAB_FILES_MAP
					__lowerCAmelCase					=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







					def __init__(  self		,     _UpperCAmelCase		,     _UpperCAmelCase=True		,     _UpperCAmelCase=True		,     _UpperCAmelCase=False		,     _UpperCAmelCase="[CLS]"		,     _UpperCAmelCase="[SEP]"		,     _UpperCAmelCase="<unk>"		,     _UpperCAmelCase="[SEP]"		,     _UpperCAmelCase="<pad>"		,     _UpperCAmelCase="[CLS]"		,     _UpperCAmelCase="[MASK]"		,     _UpperCAmelCase = None		,     **_UpperCAmelCase		,     ):
									# Mask token behave like a normal word, i.e. include the space before it and
									# is included in the raw text, there should be a match in a non-normalized sentence.
									__a	:  int         =					(
									    AddedToken(_UpperCAmelCase		,     lstrip=_UpperCAmelCase		,     rstrip=_UpperCAmelCase		,     normalized=_UpperCAmelCase       )
									    if isinstance(_UpperCAmelCase		,     _UpperCAmelCase       )
									    else mask_token
									)

									__a	:  Any         =					{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=_UpperCAmelCase		,     remove_space=_UpperCAmelCase		,     keep_accents=_UpperCAmelCase		,     bos_token=_UpperCAmelCase		,     eos_token=_UpperCAmelCase		,     unk_token=_UpperCAmelCase		,     sep_token=_UpperCAmelCase		,     pad_token=_UpperCAmelCase		,     cls_token=_UpperCAmelCase		,     mask_token=_UpperCAmelCase		,     sp_model_kwargs=self.sp_model_kwargs		,     **_UpperCAmelCase		,     )

									__a	:  Tuple         =					do_lower_case
									__a	:  Optional[Any]         =					remove_space
									__a	:  Optional[Any]         =					keep_accents
									__a	:  Union[str, Any]         =					vocab_file

									__a	:  List[str]         =					spm.SentencePieceProcessor(**self.sp_model_kwargs       )
									self.sp_model.Load(_UpperCAmelCase       )







					@property
					def 	_lowerCamelCase						(  self       ):
									return len(self.sp_model       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  Any         =					{self.convert_ids_to_tokens(_UpperCAmelCase       ): i for i in range(self.vocab_size       )}
									vocab.update(self.added_tokens_encoder       )
									return vocab







					def __getstate__(  self       ):
									__a	:  str         =					self.__dict__.copy()
									__a	:  Tuple         =					None
									return state







					def __setstate__(  self		,     _UpperCAmelCase       ):
									__a	:  Any         =					d

									# for backward compatibility
									if not hasattr(self		,     '''sp_model_kwargs'''       ):
													__a	:  Optional[Any]         =					{}

									__a	:  Dict         =					spm.SentencePieceProcessor(**self.sp_model_kwargs       )
									self.sp_model.Load(self.vocab_file       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									if self.remove_space:
													__a	:  Any         =					''' '''.join(inputs.strip().split()       )
									else:
													__a	:  Tuple         =					inputs
									__a	:  Union[str, Any]         =					outputs.replace('''``'''		,     '''"'''       ).replace('''\'\''''		,     '''"'''       )

									if not self.keep_accents:
													__a	:  List[str]         =					unicodedata.normalize('''NFKD'''		,     _UpperCAmelCase       )
													__a	:  Optional[int]         =					''''''.join([c for c in outputs if not unicodedata.combining(_UpperCAmelCase       )]       )
									if self.do_lower_case:
													__a	:  Optional[Any]         =					outputs.lower()

									return outputs







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									__a	:  int         =					self.preprocess_text(_UpperCAmelCase       )
									__a	:  Tuple         =					self.sp_model.encode(_UpperCAmelCase		,     out_type=_UpperCAmelCase       )
									__a	:  int         =					[]
									for piece in pieces:
													if len(_UpperCAmelCase       ) > 1 and piece[-1] == str(''','''       ) and piece[-2].isdigit():
																	__a	:  List[str]         =					self.sp_model.EncodeAsPieces(piece[:-1].replace(_UpperCAmelCase		,     ''''''       )       )
																	if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																					if len(cur_pieces[0]       ) == 1:
																									__a	:  Tuple         =					cur_pieces[1:]
																					else:
																									__a	:  Optional[Any]         =					cur_pieces[0][1:]
																	cur_pieces.append(piece[-1]       )
																	new_pieces.extend(_UpperCAmelCase       )
													else:
																	new_pieces.append(_UpperCAmelCase       )

									return new_pieces







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									return self.sp_model.PieceToId(_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									return self.sp_model.IdToPiece(_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									__a	:  List[str]         =					[]
									__a	:  str         =					''''''
									__a	:  Any         =					False
									for token in tokens:
													# make sure that special tokens are not decoded using sentencepiece model
													if token in self.all_special_tokens:
																	if not prev_is_special:
																					out_string += " "
																	out_string += self.sp_model.decode(_UpperCAmelCase       ) + token
																	__a	:  Tuple         =					True
																	__a	:  Tuple         =					[]
													else:
																	current_sub_tokens.append(_UpperCAmelCase       )
																	__a	:  Optional[int]         =					False
									out_string += self.sp_model.decode(_UpperCAmelCase       )
									return out_string.strip()







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase = None       ):
									__a	:  int         =					[self.sep_token_id]
									__a	:  Union[str, Any]         =					[self.cls_token_id]
									if token_ids_a is None:
													return cls + token_ids_a + sep
									return cls + token_ids_a + sep + token_ids_a + sep







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase = None		,     _UpperCAmelCase = False       ):

									if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=_UpperCAmelCase		,     token_ids_a=_UpperCAmelCase		,     already_has_special_tokens=_UpperCAmelCase       )

									if token_ids_a is not None:
													return [1] + ([0] * len(_UpperCAmelCase       )) + [1] + ([0] * len(_UpperCAmelCase       )) + [1]
									return [1] + ([0] * len(_UpperCAmelCase       )) + [1]







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase = None       ):
									__a	:  Union[str, Any]         =					[self.sep_token_id]
									__a	:  Tuple         =					[self.cls_token_id]

									if token_ids_a is None:
													return len(cls + token_ids_a + sep       ) * [0]
									return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase = None       ):
									if not os.path.isdir(_UpperCAmelCase       ):
													logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
													return
									__a	:  List[str]         =					os.path.join(
									    _UpperCAmelCase		,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

									if os.path.abspath(self.vocab_file       ) != os.path.abspath(_UpperCAmelCase       ) and os.path.isfile(self.vocab_file       ):
													copyfile(self.vocab_file		,     _UpperCAmelCase       )
									elif not os.path.isfile(self.vocab_file       ):
													with open(_UpperCAmelCase		,     '''wb'''       ) as fi:
																	__a	:  Any         =					self.sp_model.serialized_model_proto()
																	fi.write(_UpperCAmelCase       )

									return (out_vocab_file,)






"""simple docstring"""






import unittest

from transformers import SqueezeBertConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    SqueezeBertForMaskedLM,
		    SqueezeBertForMultipleChoice,
		    SqueezeBertForQuestionAnswering,
		    SqueezeBertForSequenceClassification,
		    SqueezeBertForTokenClassification,
		    SqueezeBertModel,
		)




class 			__lowercase     (     _UpperCamelCase							):







					'''simple docstring'''




					def __init__(  self		,     _UpperCAmelCase		,     _UpperCAmelCase=13		,     _UpperCAmelCase=7		,     _UpperCAmelCase=True		,     _UpperCAmelCase=True		,     _UpperCAmelCase=False		,     _UpperCAmelCase=True		,     _UpperCAmelCase=99		,     _UpperCAmelCase=32		,     _UpperCAmelCase=5		,     _UpperCAmelCase=4		,     _UpperCAmelCase=64		,     _UpperCAmelCase="gelu"		,     _UpperCAmelCase=0.1		,     _UpperCAmelCase=0.1		,     _UpperCAmelCase=512		,     _UpperCAmelCase=16		,     _UpperCAmelCase=2		,     _UpperCAmelCase=0.0_2		,     _UpperCAmelCase=3		,     _UpperCAmelCase=4		,     _UpperCAmelCase=None		,     _UpperCAmelCase=2		,     _UpperCAmelCase=2		,     _UpperCAmelCase=2		,     _UpperCAmelCase=2		,     _UpperCAmelCase=4		,     _UpperCAmelCase=1		,     ):
									__a	:  Dict         =					parent
									__a	:  str         =					batch_size
									__a	:  Union[str, Any]         =					seq_length
									__a	:  Any         =					is_training
									__a	:  int         =					use_input_mask
									__a	:  Optional[int]         =					use_token_type_ids
									__a	:  int         =					use_labels
									__a	:  int         =					vocab_size
									__a	:  int         =					hidden_size
									__a	:  str         =					num_hidden_layers
									__a	:  str         =					num_attention_heads
									__a	:  Any         =					intermediate_size
									__a	:  Union[str, Any]         =					hidden_act
									__a	:  Optional[int]         =					hidden_dropout_prob
									__a	:  str         =					attention_probs_dropout_prob
									__a	:  int         =					max_position_embeddings
									__a	:  Union[str, Any]         =					type_vocab_size
									__a	:  List[str]         =					type_sequence_label_size
									__a	:  List[str]         =					initializer_range
									__a	:  Optional[int]         =					num_labels
									__a	:  List[str]         =					num_choices
									__a	:  int         =					scope
									__a	:  Union[str, Any]         =					q_groups
									__a	:  Dict         =					k_groups
									__a	:  List[str]         =					v_groups
									__a	:  Any         =					post_attention_groups
									__a	:  Optional[int]         =					intermediate_groups
									__a	:  List[str]         =					output_groups







					def 	_lowerCamelCase						(  self       ):
									__a	:  int         =					ids_tensor([self.batch_size, self.seq_length]		,     self.vocab_size       )

									__a	:  Optional[Any]         =					None
									if self.use_input_mask:
													__a	:  int         =					random_attention_mask([self.batch_size, self.seq_length]       )

									__a	:  List[str]         =					None
									__a	:  Union[str, Any]         =					None
									__a	:  int         =					None
									if self.use_labels:
													__a	:  List[str]         =					ids_tensor([self.batch_size]		,     self.type_sequence_label_size       )
													__a	:  int         =					ids_tensor([self.batch_size, self.seq_length]		,     self.num_labels       )
													__a	:  Dict         =					ids_tensor([self.batch_size]		,     self.num_choices       )

									__a	:  int         =					self.get_config()

									return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels







					def 	_lowerCamelCase						(  self       ):
									return SqueezeBertConfig(
									    embedding_size=self.hidden_size		,     vocab_size=self.vocab_size		,     hidden_size=self.hidden_size		,     num_hidden_layers=self.num_hidden_layers		,     num_attention_heads=self.num_attention_heads		,     intermediate_size=self.intermediate_size		,     hidden_act=self.hidden_act		,     attention_probs_dropout_prob=self.hidden_dropout_prob		,     attention_dropout=self.attention_probs_dropout_prob		,     max_position_embeddings=self.max_position_embeddings		,     initializer_range=self.initializer_range		,     q_groups=self.q_groups		,     k_groups=self.k_groups		,     v_groups=self.v_groups		,     post_attention_groups=self.post_attention_groups		,     intermediate_groups=self.intermediate_groups		,     output_groups=self.output_groups		,     )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase       ):
									__a	:  Dict         =					SqueezeBertModel(config=_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  Optional[Any]         =					model(_UpperCAmelCase		,     _UpperCAmelCase       )
									__a	:  Dict         =					model(_UpperCAmelCase       )
									self.parent.assertEqual(result.last_hidden_state.shape		,     (self.batch_size, self.seq_length, self.hidden_size)       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase       ):
									__a	:  Optional[Any]         =					SqueezeBertForMaskedLM(config=_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  int         =					model(_UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     labels=_UpperCAmelCase       )
									self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.vocab_size)       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase       ):
									__a	:  int         =					SqueezeBertForQuestionAnswering(config=_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  Optional[int]         =					model(
									    _UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     start_positions=_UpperCAmelCase		,     end_positions=_UpperCAmelCase       )
									self.parent.assertEqual(result.start_logits.shape		,     (self.batch_size, self.seq_length)       )
									self.parent.assertEqual(result.end_logits.shape		,     (self.batch_size, self.seq_length)       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase       ):
									__a	:  Any         =					self.num_labels
									__a	:  List[Any]         =					SqueezeBertForSequenceClassification(_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  int         =					model(_UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     labels=_UpperCAmelCase       )
									self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_labels)       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase       ):
									__a	:  List[Any]         =					self.num_labels
									__a	:  List[str]         =					SqueezeBertForTokenClassification(config=_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()

									__a	:  Any         =					model(_UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     labels=_UpperCAmelCase       )
									self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.num_labels)       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase		,     _UpperCAmelCase       ):
									__a	:  List[str]         =					self.num_choices
									__a	:  Union[str, Any]         =					SqueezeBertForMultipleChoice(config=_UpperCAmelCase       )
									model.to(_UpperCAmelCase       )
									model.eval()
									__a	:  int         =					input_ids.unsqueeze(1       ).expand(-1		,     self.num_choices		,     -1       ).contiguous()
									__a	:  Optional[Any]         =					input_mask.unsqueeze(1       ).expand(-1		,     self.num_choices		,     -1       ).contiguous()
									__a	:  Union[str, Any]         =					model(
									    _UpperCAmelCase		,     attention_mask=_UpperCAmelCase		,     labels=_UpperCAmelCase		,     )
									self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_choices)       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  Any         =					self.prepare_config_and_inputs()
									((__a)    ,  (__a)    ,  (__a)    ,  (__a)    ,  (__a)    ,  (__a))	:  Optional[Any]         =					config_and_inputs
									__a	:  List[Any]         =					{'''input_ids''': input_ids, '''attention_mask''': input_mask}
									return config, inputs_dict






@require_torch
class 			__lowercase     (     _UpperCamelCase				, _UpperCamelCase				, unittest.TestCase							):







					'''simple docstring'''




					__lowerCAmelCase					=					(
					    (
					        SqueezeBertModel,
					        SqueezeBertForMaskedLM,
					        SqueezeBertForMultipleChoice,
					        SqueezeBertForQuestionAnswering,
					        SqueezeBertForSequenceClassification,
					        SqueezeBertForTokenClassification,
					    )
					    if is_torch_available()
					    else None
					)
					__lowerCAmelCase					=					(
					    {
					        '''feature-extraction''': SqueezeBertModel,
					        '''fill-mask''': SqueezeBertForMaskedLM,
					        '''question-answering''': SqueezeBertForQuestionAnswering,
					        '''text-classification''': SqueezeBertForSequenceClassification,
					        '''token-classification''': SqueezeBertForTokenClassification,
					        '''zero-shot''': SqueezeBertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					__lowerCAmelCase					=					False
					__lowerCAmelCase					=					True
					__lowerCAmelCase					=					False







					def 	_lowerCamelCase						(  self       ):
									__a	:  Union[str, Any]         =					SqueezeBertModelTester(self       )
									__a	:  Dict         =					ConfigTester(self		,     config_class=_UpperCAmelCase		,     dim=37       )







					def 	_lowerCamelCase						(  self       ):
									self.config_tester.run_common_tests()







					def 	_lowerCamelCase						(  self       ):
									__a	:  Tuple         =					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_squeezebert_model(*_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  List[Any]         =					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_squeezebert_for_masked_lm(*_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  str         =					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_squeezebert_for_question_answering(*_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  Optional[Any]         =					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_squeezebert_for_sequence_classification(*_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  Optional[int]         =					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_squeezebert_for_token_classification(*_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  List[str]         =					self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_squeezebert_for_multiple_choice(*_UpperCAmelCase       )







					@slow
					def 	_lowerCamelCase						(  self       ):
									for model_name in SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													__a	:  Any         =					SqueezeBertModel.from_pretrained(_UpperCAmelCase       )
													self.assertIsNotNone(_UpperCAmelCase       )






@require_sentencepiece
@require_tokenizers
@require_torch
class 			__lowercase     (     unittest.TestCase							):







					'''simple docstring'''




					@slow
					def 	_lowerCamelCase						(  self       ):
									__a	:  int         =					SqueezeBertForSequenceClassification.from_pretrained('''squeezebert/squeezebert-mnli'''       )

									__a	:  Tuple         =					torch.tensor([[1, 29414, 232, 328, 740, 1140, 12695, 69, 13, 1588, 2]]       )
									__a	:  List[str]         =					model(_UpperCAmelCase       )[0]
									__a	:  int         =					torch.Size((1, 3)       )
									self.assertEqual(output.shape		,     _UpperCAmelCase       )
									__a	:  int         =					torch.tensor([[0.6_4_0_1, -0.0_3_4_9, -0.6_0_4_1]]       )
									self.assertTrue(torch.allclose(_UpperCAmelCase		,     _UpperCAmelCase		,     atol=1e-4       )       )