


'''simple docstring'''
import tempfile
import unittest

from transformers import TaConfig, is_torch_available
from transformers.testing_utils import (
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
    torch_device,
)

from ...generation.test_utils import GenerationTesterMixin
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import AutoTokenizer, UMTaForConditionalGeneration, UMTaForQuestionAnswering, UMTaModel
class        lowercase     :

		"""simple docstring"""

		def __init__(		self   ,       UpperCamelCase_   ,       UpperCamelCase_=99   ,       UpperCamelCase_=13   ,       UpperCamelCase_=7   ,       UpperCamelCase_=9   ,       UpperCamelCase_=True   ,       UpperCamelCase_=True   ,       UpperCamelCase_=False   ,       UpperCamelCase_=32   ,       UpperCamelCase_=5   ,       UpperCamelCase_=4   ,       UpperCamelCase_=37   ,       UpperCamelCase_=8   ,       UpperCamelCase_=0.1   ,       UpperCamelCase_=0.002   ,       UpperCamelCase_=1   ,       UpperCamelCase_=0   ,       UpperCamelCase_=0   ,       UpperCamelCase_=None   ,       UpperCamelCase_=None   ,       ):

							'''simple docstring'''


							UpperCamelCase__       :Union[str, Any]							 =			parent
							UpperCamelCase__       :Optional[Any]							 =			batch_size
							UpperCamelCase__       :str							 =			encoder_seq_length
							UpperCamelCase__       :Optional[Any]							 =			decoder_seq_length
							# For common tests
							UpperCamelCase__       :List[str]							 =			self.decoder_seq_length
							UpperCamelCase__       :Optional[int]							 =			is_training
							UpperCamelCase__       :str							 =			use_attention_mask
							UpperCamelCase__       :List[Any]							 =			use_labels
							UpperCamelCase__       :Optional[Any]							 =			vocab_size
							UpperCamelCase__       :List[Any]							 =			hidden_size
							UpperCamelCase__       :Dict							 =			num_hidden_layers
							UpperCamelCase__       :Any							 =			num_attention_heads
							UpperCamelCase__       :Optional[int]							 =			d_ff
							UpperCamelCase__       :Optional[Any]							 =			relative_attention_num_buckets
							UpperCamelCase__       :List[str]							 =			dropout_rate
							UpperCamelCase__       :Union[str, Any]							 =			initializer_factor
							UpperCamelCase__       :Any							 =			eos_token_id
							UpperCamelCase__       :str							 =			pad_token_id
							UpperCamelCase__       :str							 =			decoder_start_token_id
							UpperCamelCase__       :int							 =			None
							UpperCamelCase__       :str							 =			decoder_layers




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return TaConfig.from_pretrained('''google/umt5-base'''							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_=None   ,       UpperCamelCase_=None   ,       UpperCamelCase_=None   ,       UpperCamelCase_=None   ,       UpperCamelCase_=None   ,       ):

							'''simple docstring'''


							if attention_mask is None:
												UpperCamelCase__       :Union[str, Any]							 =			input_ids.ne(config.pad_token_id							)
							if decoder_attention_mask is None:
												UpperCamelCase__       :Any							 =			decoder_input_ids.ne(config.pad_token_id							)
							if head_mask is None:
												UpperCamelCase__       :Dict							 =			torch.ones(config.num_hidden_layers   ,       config.num_attention_heads   ,       device=UpperCamelCase_							)
							if decoder_head_mask is None:
												UpperCamelCase__       :Optional[int]							 =			torch.ones(config.num_decoder_layers   ,       config.num_attention_heads   ,       device=UpperCamelCase_							)
							if cross_attn_head_mask is None:
												UpperCamelCase__       :Any							 =			torch.ones(
												    config.num_decoder_layers   ,       config.num_attention_heads   ,       device=UpperCamelCase_							)
							return {
							    "input_ids": input_ids,
							    "decoder_input_ids": decoder_input_ids,
							    "attention_mask": attention_mask,
							    "decoder_attention_mask": decoder_attention_mask,
							    "head_mask": head_mask,
							    "decoder_head_mask": decoder_head_mask,
							    "cross_attn_head_mask": cross_attn_head_mask,
							}




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :List[str]							 =			ids_tensor([self.batch_size, self.encoder_seq_length]   ,       self.vocab_size							)
							UpperCamelCase__       :List[str]							 =			ids_tensor([self.batch_size, self.decoder_seq_length]   ,       self.vocab_size							)

							# we need to clamp the input ids here to avoid having pad token in between
							# this is because for NllbMoe the position_ids are prepared such that
							# all pad tokens have pos id = 2 and rest are between 2..seq_length
							# and the seq_length here is seq_length - num_pad_tokens
							# but when using past, there is no way of knowing if the past input ids had
							# pad tokens in them, which results in incorrect seq_lenth and which in turn results in
							# position_ids being off by num_pad_tokens in past input
							UpperCamelCase__       :Tuple							 =			input_ids.clamp(self.pad_token_id + 1							)
							UpperCamelCase__       :Tuple							 =			decoder_input_ids.clamp(self.pad_token_id + 1							)

							UpperCamelCase__       :int							 =			self.get_config()
							UpperCamelCase__       :int							 =			config.num_attention_heads
							UpperCamelCase__       :Union[str, Any]							 =			self.prepare_inputs_dict(UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_							)
							return config, input_dict




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__ ,    UpperCamelCase__       :int							 =			self.prepare_config_and_inputs()
							return config, inputs_dict




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return TaConfig(
							    vocab_size=166   ,       d_model=self.hidden_size   ,       d_ff=self.d_ff   ,       d_kv=self.hidden_size // self.num_attention_heads   ,       num_layers=self.num_hidden_layers   ,       num_decoder_layers=self.decoder_layers   ,       num_heads=self.num_attention_heads   ,       relative_attention_num_buckets=self.relative_attention_num_buckets   ,       dropout_rate=self.dropout_rate   ,       initializer_factor=self.initializer_factor   ,       eos_token_id=self.eos_token_id   ,       bos_token_id=self.pad_token_id   ,       pad_token_id=self.pad_token_id   ,       decoder_start_token_id=self.decoder_start_token_id   ,       )




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return TaConfig(
							    vocab_size=self.vocab_size   ,       d_model=self.hidden_size   ,       d_ff=self.d_ff   ,       d_kv=self.hidden_size // self.num_attention_heads   ,       num_layers=self.num_hidden_layers   ,       num_decoder_layers=self.decoder_layers   ,       num_heads=self.num_attention_heads   ,       relative_attention_num_buckets=self.relative_attention_num_buckets   ,       dropout_rate=self.dropout_rate   ,       initializer_factor=self.initializer_factor   ,       eos_token_id=self.eos_token_id   ,       bos_token_id=self.pad_token_id   ,       pad_token_id=self.pad_token_id   ,       decoder_start_token_id=self.decoder_start_token_id   ,       )




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       ):

							'''simple docstring'''


							UpperCamelCase__       :Dict							 =			UMTaModel(config=UpperCamelCase_							)
							model.to(UpperCamelCase_							)
							model.eval()
							UpperCamelCase__       :Tuple							 =			model(
							    input_ids=UpperCamelCase_   ,       decoder_input_ids=UpperCamelCase_   ,       attention_mask=UpperCamelCase_   ,       decoder_attention_mask=UpperCamelCase_   ,       )
							UpperCamelCase__       :Tuple							 =			model(input_ids=UpperCamelCase_   ,       decoder_input_ids=UpperCamelCase_							)
							UpperCamelCase__       :Tuple							 =			result.last_hidden_state
							UpperCamelCase__       :Union[str, Any]							 =			result.past_key_values
							UpperCamelCase__       :List[str]							 =			result.encoder_last_hidden_state

							self.parent.assertEqual(encoder_output.size()   ,       (self.batch_size, self.encoder_seq_length, self.hidden_size)							)
							self.parent.assertEqual(decoder_output.size()   ,       (self.batch_size, self.decoder_seq_length, self.hidden_size)							)
							# There should be `num_layers` key value embeddings stored in decoder_past
							self.parent.assertEqual(len(UpperCamelCase_							)   ,       config.num_layers							)
							# There should be a self attn key, a self attn value, a cross attn key and a cross attn value stored in each decoder_past tuple
							self.parent.assertEqual(len(decoder_past[0]							)   ,       4							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       ):

							'''simple docstring'''


							UpperCamelCase__       :str							 =			UMTaModel(config=UpperCamelCase_							).get_decoder().to(UpperCamelCase_							).eval()
							# first forward pass
							UpperCamelCase__       :Any							 =			model(UpperCamelCase_   ,       use_cache=UpperCamelCase_							)
							UpperCamelCase__       :int							 =			model(UpperCamelCase_							)
							UpperCamelCase__       :Optional[Any]							 =			model(UpperCamelCase_   ,       use_cache=UpperCamelCase_							)

							self.parent.assertTrue(len(UpperCamelCase_							) == len(UpperCamelCase_							)							)
							self.parent.assertTrue(len(UpperCamelCase_							) == len(UpperCamelCase_							) + 1							)

							UpperCamelCase__ ,    UpperCamelCase__       :Dict							 =			outputs.to_tuple()

							# create hypothetical next token and extent to next_input_ids
							UpperCamelCase__       :List[str]							 =			ids_tensor((self.batch_size, 1)   ,       config.vocab_size							)

							# append to next input_ids and
							UpperCamelCase__       :Union[str, Any]							 =			torch.cat([input_ids, next_tokens]   ,       dim=-1							)

							UpperCamelCase__       :List[str]							 =			model(UpperCamelCase_							)['''last_hidden_state''']
							UpperCamelCase__       :Any							 =			model(UpperCamelCase_   ,       past_key_values=UpperCamelCase_							)['''last_hidden_state''']

							# select random slice
							UpperCamelCase__       :int							 =			ids_tensor((1,)   ,       output_from_past.shape[-1]							).item()
							UpperCamelCase__       :Union[str, Any]							 =			output_from_no_past[:, -1, random_slice_idx].detach()
							UpperCamelCase__       :List[str]							 =			output_from_past[:, 0, random_slice_idx].detach()

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(UpperCamelCase_   ,       UpperCamelCase_   ,       atol=1e-3							)							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       ):

							'''simple docstring'''


							UpperCamelCase__       :Optional[int]							 =			UMTaModel(config=UpperCamelCase_							).to(UpperCamelCase_							).half().eval()
							UpperCamelCase__       :Tuple							 =			model(**UpperCamelCase_							)['''last_hidden_state''']
							self.parent.assertFalse(torch.isnan(UpperCamelCase_							).any().item()							)



@require_torch
class        lowercase     (							A__     ,		A__     ,		A__     ,		unittest.TestCase ):

		"""simple docstring"""

		_a   			=    (
		    (UMTaModel, UMTaForConditionalGeneration, UMTaForQuestionAnswering) if is_torch_available() else ()
		)
		_a   			=    (UMTaForConditionalGeneration,) if is_torch_available() else ()
		_a   			=    (
		    {
		        'conversational': UMTaForConditionalGeneration,
		        'feature-extraction': UMTaModel,
		        'summarization': UMTaForConditionalGeneration,
		        'text2text-generation': UMTaForConditionalGeneration,
		        'translation': UMTaForConditionalGeneration,
		        'question-answering': UMTaForQuestionAnswering,
		    }
		    if is_torch_available()
		    else {}
		)
		_a   			=    True
		_a   			=    False
		_a   			=    False
		_a   			=    True
		_a   			=    True
		# The small UMT5 model needs higher percentages for CPU/MP tests
		_a   			=    [0.8, 0.9]




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			UMTaModelTester(self							)




		@unittest.skip('''Test has a segmentation fault on torch 1.8.0'''							)
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Any							 =			self.model_tester.prepare_config_and_inputs()
							UpperCamelCase__       :Optional[Any]							 =			UMTaModel(config_and_inputs[0]							).to(UpperCamelCase_							)
							with tempfile.TemporaryDirectory() as tmpdirname:
												torch.onnx.export(
												    UpperCamelCase_   ,       (config_and_inputs[1], config_and_inputs[3], config_and_inputs[2])   ,       F'''{tmpdirname}/t5_test.onnx'''   ,       export_params=UpperCamelCase_   ,       opset_version=9   ,       input_names=['''input_ids''', '''decoder_input_ids''']   ,       )




		@unittest.skipIf(torch_device == '''cpu'''   ,       '''Cant do half precision'''							)
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[Any]							 =			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model_fpaa_forward(*UpperCamelCase_							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Union[str, Any]							 =			['''encoder_attentions''', '''decoder_attentions''', '''cross_attentions''']
							UpperCamelCase__       :List[Any]							 =			self.model_tester.prepare_config_and_inputs()
							UpperCamelCase__       :str							 =			config_and_inputs[0]
							UpperCamelCase__       :str							 =			UMTaForConditionalGeneration(UpperCamelCase_							).eval()
							model.to(UpperCamelCase_							)

							UpperCamelCase__       :str							 =			{
							    '''head_mask''': torch.zeros(config.num_layers   ,       config.num_heads   ,       device=UpperCamelCase_							),
							    '''decoder_head_mask''': torch.zeros(config.num_decoder_layers   ,       config.num_heads   ,       device=UpperCamelCase_							),
							    '''cross_attn_head_mask''': torch.zeros(config.num_decoder_layers   ,       config.num_heads   ,       device=UpperCamelCase_							),
							}

							for attn_name, (name, mask) in zip(UpperCamelCase_   ,       head_masking.items()							):
												UpperCamelCase__       :List[str]							 =			{name: mask}
												# Explicitly pass decoder_head_mask as it is required from T5 model when head_mask specified
												if name == "head_mask":
																	UpperCamelCase__       :Tuple							 =			torch.ones(
																	    config.num_decoder_layers   ,       config.num_heads   ,       device=UpperCamelCase_							)

												UpperCamelCase__       :int							 =			model.generate(
												    config_and_inputs[1]['''input_ids''']   ,       num_beams=1   ,       max_length=3   ,       output_attentions=UpperCamelCase_   ,       return_dict_in_generate=UpperCamelCase_   ,       **UpperCamelCase_   ,       )
												# We check the state of decoder_attentions and cross_attentions just from the last step
												UpperCamelCase__       :Optional[Any]							 =			out[attn_name] if attn_name == attention_names[0] else out[attn_name][-1]
												self.assertEqual(sum([w.sum().item() for w in attn_weights]							)   ,       0.0							)




		@unittest.skip('''Does not work on the tiny model as we keep hitting edge cases.'''							)
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							pass



@require_torch
@require_sentencepiece
@require_tokenizers
class        lowercase     (							unittest.TestCase ):

		"""simple docstring"""

		@slow
		@unittest.skip(
		    '''Unless we stop stripping left and right by default for all special tokens, the expected ids obtained here will not match the original ones. Wait for https://github.com/huggingface/transformers/pull/23909 to be merged'''							)
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :int							 =			UMTaForConditionalGeneration.from_pretrained('''google/umt5-small'''   ,       return_dict=UpperCamelCase_							).to(UpperCamelCase_							)
							UpperCamelCase__       :List[Any]							 =			AutoTokenizer.from_pretrained('''google/umt5-small'''   ,       use_fast=UpperCamelCase_   ,       legacy=UpperCamelCase_							)
							UpperCamelCase__       :Optional[int]							 =			[
							    '''Bonjour monsieur <extra_id_0> bien <extra_id_1>.''',
							    '''No se como puedo <extra_id_0>.''',
							    '''This is the reason why we <extra_id_0> them.''',
							    '''The <extra_id_0> walks in <extra_id_1>, seats''',
							    '''A <extra_id_0> walks into a bar and orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>.''',
							]
							UpperCamelCase__       :Union[str, Any]							 =			tokenizer(UpperCamelCase_   ,       return_tensors='''pt'''   ,       padding=UpperCamelCase_							).input_ids
							# fmt: off
							UpperCamelCase__       :Optional[Any]							 =			torch.tensor(
							    [
							        [ 38530, 210703, 256299, 1410, 256298, 274, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
							        [   826, 321, 671, 25922, 256299, 274, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
							        [  1460, 339, 312, 19014, 10620, 758, 256299, 2355,274, 1, 0, 0, 0, 0, 0, 0,0, 0],
							        [   517, 256299, 14869, 281, 301, 256298, 275, 119983,1, 0, 0, 0, 0, 0, 0, 0,0, 0],
							        [   320, 256299, 14869, 281, 2234, 289, 2275, 333,61391, 289, 256298, 543, 256297, 168714, 329, 256296,274, 1],
							    ]							)
							# fmt: on
							torch.testing.assert_allclose(UpperCamelCase_   ,       UpperCamelCase_							)

							UpperCamelCase__       :int							 =			model.generate(input_ids.to(UpperCamelCase_							)							)
							UpperCamelCase__       :str							 =			[
							    '''<pad><extra_id_0> et<extra_id_1> [eod] <extra_id_2><extra_id_55>.. [eod] 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 <extra_id_56>ajšietosto<extra_id_56>lleux<extra_id_19><extra_id_6>ajšie</s>''',
							    '''<pad><extra_id_0>.<extra_id_1>.,<0x0A>...spech <0x0A><extra_id_20> <extra_id_21></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
							    '''<pad><extra_id_0> are not going to be a part of the world. We are not going to be a part of<extra_id_1> and<extra_id_2><0x0A><extra_id_48>.<extra_id_48></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
							    '''<pad><extra_id_0> door<extra_id_1>, the door<extra_id_2> 피해[/</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
							    '''<pad><extra_id_0>nyone who<extra_id_1> drink<extra_id_2> a<extra_id_3> alcohol<extra_id_4> A<extra_id_5> A. This<extra_id_6> I<extra_id_7><extra_id_52><extra_id_53></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
							]
							UpperCamelCase__       :List[str]							 =			tokenizer.batch_decode(UpperCamelCase_							)
							self.assertEqual(UpperCamelCase_   ,       UpperCamelCase_							)



'''simple docstring'''
import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
		import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case											=					logging.get_logger(__name__)

__snake_case											=					{'''vocab_file''': '''vocab.txt'''}

__snake_case											=					{
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

__snake_case											=					{
    '''openbmb/cpm-ant-10b''': 1024,
}


def       a				(      __a       )  ->     Tuple:



					'''simple docstring'''






					UpperCamelCase__       :List[str]							 =			collections.OrderedDict()
					with open(__a			, '''r'''			, encoding='''utf-8'''       ) as reader:
										UpperCamelCase__       :Dict							 =			reader.readlines()
					for index, token in enumerate(__a       ):
										UpperCamelCase__       :str							 =			token.rstrip('''\n'''       )
										UpperCamelCase__       :Optional[int]							 =			index
					return vocab
class        lowercase     (							A__ ):

		"""simple docstring"""

		def __init__(		self   ,       UpperCamelCase_   ,       UpperCamelCase_="<unk>"   ,       UpperCamelCase_=200							):

							'''simple docstring'''


							UpperCamelCase__       :Tuple							 =			vocab
							UpperCamelCase__       :List[str]							 =			unk_token
							UpperCamelCase__       :Tuple							 =			max_input_chars_per_word




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[int]							 =			list(UpperCamelCase_							)
							if len(UpperCamelCase_							) > self.max_input_chars_per_word:
												return [self.unk_token]

							UpperCamelCase__       :List[Any]							 =			0
							UpperCamelCase__       :str							 =			[]
							while start < len(UpperCamelCase_							):
												UpperCamelCase__       :int							 =			len(UpperCamelCase_							)
												UpperCamelCase__       :List[Any]							 =			None
												while start < end:
																	UpperCamelCase__       :int							 =			''''''.join(chars[start:end]							)
																	if substr in self.vocab:
																						UpperCamelCase__       :List[Any]							 =			substr
																						break
																	end -= 1
												if cur_substr is None:
																	sub_tokens.append(self.unk_token							)
																	start += 1
												else:
																	sub_tokens.append(UpperCamelCase_							)
																	UpperCamelCase__       :Any							 =			end

							return sub_tokens



class        lowercase     (							A__ ):

		"""simple docstring"""

		_a   			=    VOCAB_FILES_NAMES
		_a   			=    PRETRAINED_VOCAB_FILES_MAP
		_a   			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a   			=    ['input_ids', 'attention_mask']
		_a   			=    False




		def __init__(		self   ,       UpperCamelCase_   ,       UpperCamelCase_="<d>"   ,       UpperCamelCase_="</d>"   ,       UpperCamelCase_="<s>"   ,       UpperCamelCase_="</s>"   ,       UpperCamelCase_="<pad>"   ,       UpperCamelCase_="<unk>"   ,       UpperCamelCase_="</n>"   ,       UpperCamelCase_="</_>"   ,       UpperCamelCase_="left"   ,       **UpperCamelCase_   ,       ):

							'''simple docstring'''


							requires_backends(self   ,       ['''jieba''']							)
							super().__init__(
							    bod_token=UpperCamelCase_   ,       eod_token=UpperCamelCase_   ,       bos_token=UpperCamelCase_   ,       eos_token=UpperCamelCase_   ,       pad_token=UpperCamelCase_   ,       unk_token=UpperCamelCase_   ,       line_token=UpperCamelCase_   ,       space_token=UpperCamelCase_   ,       padding_side=UpperCamelCase_   ,       **UpperCamelCase_   ,       )
							UpperCamelCase__       :Tuple							 =			bod_token
							UpperCamelCase__       :Dict							 =			eod_token
							UpperCamelCase__       :Optional[int]							 =			load_vocab(UpperCamelCase_							)
							UpperCamelCase__       :Tuple							 =			self.encoder[space_token]
							UpperCamelCase__       :List[Any]							 =			self.encoder[line_token]

							del self.encoder[space_token]
							del self.encoder[line_token]

							UpperCamelCase__       :Union[str, Any]							 =			collections.OrderedDict(sorted(self.encoder.items()   ,       key=lambda UpperCamelCase_							: x[1]							)							)
							UpperCamelCase__       :Union[str, Any]							 =			{v: k for k, v in self.encoder.items()}

							UpperCamelCase__       :List[Any]							 =			WordpieceTokenizer(vocab=self.encoder   ,       unk_token=self.unk_token							)




		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return self.encoder[self.bod_token]




		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return self.encoder[self.eod_token]




		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return self.encoder["\n"]




		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return len(self.encoder							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return dict(self.encoder   ,       **self.added_tokens_encoder							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			[]
							for x in jieba.cut(UpperCamelCase_   ,       cut_all=UpperCamelCase_							):
												output_tokens.extend(self.wordpiece_tokenizer.tokenize(UpperCamelCase_							)							)
							return output_tokens




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       **UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			[i for i in token_ids if i >= 0]
							UpperCamelCase__       :Optional[int]							 =			[
							    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
							]
							return super()._decode(UpperCamelCase_   ,       **UpperCamelCase_							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return token in self.encoder




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return "".join(UpperCamelCase_							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return self.encoder.get(UpperCamelCase_   ,       self.encoder.get(self.unk_token							)							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return self.decoder.get(UpperCamelCase_   ,       self.unk_token							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None							):

							'''simple docstring'''


							if os.path.isdir(UpperCamelCase_							):
												UpperCamelCase__       :int							 =			os.path.join(
												    UpperCamelCase_   ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
							else:
												UpperCamelCase__       :str							 =			(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
							UpperCamelCase__       :Any							 =			0
							if " " in self.encoder:
												UpperCamelCase__       :Dict							 =			self.encoder[''' ''']
												del self.encoder[" "]
							if "\n" in self.encoder:
												UpperCamelCase__       :List[str]							 =			self.encoder['''\n''']
												del self.encoder["\n"]
							UpperCamelCase__       :List[str]							 =			collections.OrderedDict(sorted(self.encoder.items()   ,       key=lambda UpperCamelCase_							: x[1]							)							)
							with open(UpperCamelCase_   ,       '''w'''   ,       encoding='''utf-8'''							) as writer:
												for token, token_index in self.encoder.items():
																	if index != token_index:
																						logger.warning(
																						    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																						    ''' Please check that the vocabulary is not corrupted!'''							)
																						UpperCamelCase__       :Any							 =			token_index
																	writer.write(token + '''\n'''							)
																	index += 1
							return (vocab_file,)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None							):

							'''simple docstring'''


							if token_ids_a is None:
												return [self.bos_token_id] + token_ids_a
							return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None   ,       UpperCamelCase_ = False							):

							'''simple docstring'''


							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=UpperCamelCase_   ,       token_ids_a=UpperCamelCase_   ,       already_has_special_tokens=UpperCamelCase_							)

							if token_ids_a is not None:
												return [1] + ([0] * len(UpperCamelCase_							)) + [1] + ([0] * len(UpperCamelCase_							))
							return [1] + ([0] * len(UpperCamelCase_							))