





'''simple docstring'''



from queue import Queue
from typing import TYPE_CHECKING, Optional


if TYPE_CHECKING:
	from ..models.auto import AutoTokenizer






class 							__lowercase    :







	'''simple docstring'''

	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Optional[int]:







				'''simple docstring'''





				raise NotImplementedError()


	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				raise NotImplementedError()






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	def __init__(self       ,_lowerCamelCase       ,_lowerCamelCase = False       ,**_lowerCamelCase	)							->							str:







				'''simple docstring'''





				__lowercase			  =							tokenizer
				__lowercase			  =							skip_prompt
				__lowercase			  =							decode_kwargs

				# variables used in the streaming process
				__lowercase			  =							[]
				__lowercase			  =							0
				__lowercase			  =							True


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							str:







				'''simple docstring'''





				if len(value.shape	) > 1 and value.shape[0] > 1:
							raise ValueError('''TextStreamer only supports batch size 1'''	)
				elif len(value.shape	) > 1:
							__lowercase			  =							value[0]

				if self.skip_prompt and self.next_tokens_are_prompt:
							__lowercase			  =							False
							return

				# Add the new token to the cache and decodes the entire thing.
				self.token_cache.extend(value.tolist()	)
				__lowercase			  =							self.tokenizer.decode(self.token_cache       ,**self.decode_kwargs	)

				# After the symbol for a new line, we flush the cache.
				if text.endswith('''\n'''	):
							__lowercase			  =							text[self.print_len :]
							__lowercase			  =							[]
							__lowercase			  =							0
				# If the last token is a CJK character, we print the characters.
				elif len(_lowerCamelCase	) > 0 and self._is_chinese_char(ord(text[-1]	)	):
							__lowercase			  =							text[self.print_len :]
							self.print_len += len(_lowerCamelCase	)
				# Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
				# which may change with the subsequent token -- there are probably smarter ways to do this!)
				else:
							__lowercase			  =							text[self.print_len : text.rfind(''' '''	) + 1]
							self.print_len += len(_lowerCamelCase	)

				self.on_finalized_text(_lowerCamelCase	)


	def   _UpperCAmelCase       (self	)							->							Union[str, Any]:







				'''simple docstring'''





				if len(self.token_cache	) > 0:
							__lowercase			  =							self.tokenizer.decode(self.token_cache       ,**self.decode_kwargs	)
							__lowercase			  =							text[self.print_len :]
							__lowercase			  =							[]
							__lowercase			  =							0
				else:
							__lowercase			  =							''''''

				__lowercase			  =							True
				self.on_finalized_text(_lowerCamelCase       ,stream_end=_lowerCamelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = False	)							->							Any:







				'''simple docstring'''





				print(_lowerCamelCase       ,flush=_lowerCamelCase       ,end='''''' if not stream_end else None	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Optional[int]:







				'''simple docstring'''





				if (
				    (cp >= 0x4_e_0_0 and cp <= 0x9_f_f_f)
				    or (cp >= 0x3_4_0_0 and cp <= 0x4_d_b_f)  #
				    or (cp >= 0x2_0_0_0_0 and cp <= 0x2_a_6_d_f)  #
				    or (cp >= 0x2_a_7_0_0 and cp <= 0x2_b_7_3_f)  #
				    or (cp >= 0x2_b_7_4_0 and cp <= 0x2_b_8_1_f)  #
				    or (cp >= 0x2_b_8_2_0 and cp <= 0x2_c_e_a_f)  #
				    or (cp >= 0xf_9_0_0 and cp <= 0xf_a_f_f)
				    or (cp >= 0x2_f_8_0_0 and cp <= 0x2_f_a_1_f)  #
				):  #
							return True

				return False






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	def __init__(self       ,_lowerCamelCase       ,_lowerCamelCase = False       ,_lowerCamelCase = None       ,**_lowerCamelCase	)							->							Tuple:







				'''simple docstring'''





				super().__init__(_lowerCamelCase       ,_lowerCamelCase       ,**_lowerCamelCase	)
				__lowercase			  =							Queue()
				__lowercase			  =							None
				__lowercase			  =							timeout


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = False	)							->							Any:







				'''simple docstring'''





				self.text_queue.put(_lowerCamelCase       ,timeout=self.timeout	)
				if stream_end:
							self.text_queue.put(self.stop_signal       ,timeout=self.timeout	)


	def __iter__(self	)							->							List[str]:







				'''simple docstring'''





				return self


	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				__lowercase			  =							self.text_queue.get(timeout=self.timeout	)
				if value == self.stop_signal:
							raise StopIteration()
				else:
							return value












'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_SCREAMING_SNAKE_CASE   =      {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_SCREAMING_SNAKE_CASE   =      {
    '''vocab_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-generator''': '''https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt''',
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-generator''': (
            '''https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'''
        ),
    },
}

_SCREAMING_SNAKE_CASE   =      {
    '''google/electra-small-generator''': 5_1_2,
    '''google/electra-base-generator''': 5_1_2,
    '''google/electra-large-generator''': 5_1_2,
    '''google/electra-small-discriminator''': 5_1_2,
    '''google/electra-base-discriminator''': 5_1_2,
    '''google/electra-large-discriminator''': 5_1_2,
}

_SCREAMING_SNAKE_CASE   =      {
    '''google/electra-small-generator''': {'''do_lower_case''': True},
    '''google/electra-base-generator''': {'''do_lower_case''': True},
    '''google/electra-large-generator''': {'''do_lower_case''': True},
    '''google/electra-small-discriminator''': {'''do_lower_case''': True},
    '''google/electra-base-discriminator''': {'''do_lower_case''': True},
    '''google/electra-large-discriminator''': {'''do_lower_case''': True},
}






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	a      :			str     =    VOCAB_FILES_NAMES
	a      :			Dict     =    PRETRAINED_VOCAB_FILES_MAP
	a      :			int     =    PRETRAINED_INIT_CONFIGURATION
	a      :			Tuple     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a      :			List[Any]     =    ElectraTokenizer


	def __init__(self       ,_lowerCamelCase=None       ,_lowerCamelCase=None       ,_lowerCamelCase=True       ,_lowerCamelCase="[UNK]"       ,_lowerCamelCase="[SEP]"       ,_lowerCamelCase="[PAD]"       ,_lowerCamelCase="[CLS]"       ,_lowerCamelCase="[MASK]"       ,_lowerCamelCase=True       ,_lowerCamelCase=None       ,**_lowerCamelCase       ,)							->							List[Any]:







				'''simple docstring'''





				super().__init__(
				    _lowerCamelCase       ,tokenizer_file=_lowerCamelCase       ,do_lower_case=_lowerCamelCase       ,unk_token=_lowerCamelCase       ,sep_token=_lowerCamelCase       ,pad_token=_lowerCamelCase       ,cls_token=_lowerCamelCase       ,mask_token=_lowerCamelCase       ,tokenize_chinese_chars=_lowerCamelCase       ,strip_accents=_lowerCamelCase       ,**_lowerCamelCase       ,)

				__lowercase			  =							json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
				if (
				    normalizer_state.get('''lowercase'''       ,_lowerCamelCase	) != do_lower_case
				    or normalizer_state.get('''strip_accents'''       ,_lowerCamelCase	) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''       ,_lowerCamelCase	) != tokenize_chinese_chars
				):
							__lowercase			  =							getattr(_lowerCamelCase       ,normalizer_state.pop('''type'''	)	)
							__lowercase			  =							do_lower_case
							__lowercase			  =							strip_accents
							__lowercase			  =							tokenize_chinese_chars
							__lowercase			  =							normalizer_class(**_lowerCamelCase	)

				__lowercase			  =							do_lower_case


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase=None	)							->							str:







				'''simple docstring'''





				__lowercase			  =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							List[int]:







				'''simple docstring'''





				__lowercase			  =							[self.sep_token_id]
				__lowercase			  =							[self.cls_token_id]
				if token_ids_a is None:
							return len(cls + token_ids_a + sep	) * [0]
				return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							Tuple[str]:







				'''simple docstring'''





				__lowercase			  =							self._tokenizer.model.save(_lowerCamelCase       ,name=_lowerCamelCase	)
				return tuple(_lowerCamelCase	)





