




"""simple docstring"""


import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin


class _lowerCamelCase    (					a_       ,	unittest.TestCase						):
   _lowerCamelCase    :Optional[Any]     				=     TransfoXLTokenizer
   _lowerCamelCase    :Optional[int]     				=     False
   _lowerCamelCase    :Dict     				=     False






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						Optional[Any]:

         """simple docstring"""
         super().setUp()

         lowerCAmelCase__      :			List[str]   =	[
             """<unk>""",
             """[CLS]""",
             """[SEP]""",
             """want""",
             """unwanted""",
             """wa""",
             """un""",
             """running""",
             """,""",
             """low""",
             """l""",
         ]
         lowerCAmelCase__      :			str   =	os.path.join(self.tmpdirname   ,     VOCAB_FILES_NAMES["""vocab_file"""]   )
         with open(self.vocab_file   ,     """w"""   ,     encoding="""utf-8"""   ) as vocab_writer:
               vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]   )   )






   def 	_lowerCAmelCase   (		self       :		int   ,     **UpperCamelCase       :		Any   )			->						List[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	True
         return TransfoXLTokenizer.from_pretrained(self.tmpdirname   ,     **UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   ,     UpperCamelCase       :		int   )			->						Dict:

         """simple docstring"""
         lowerCAmelCase__      :			List[str]   =	"""<unk> UNwanted , running"""
         lowerCAmelCase__      :			Any   =	"""<unk> unwanted, running"""
         return input_text, output_text






   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[int]   =	TransfoXLTokenizer(vocab_file=self.vocab_file   ,     lower_case=UpperCamelCase   )

         lowerCAmelCase__      :			Any   =	tokenizer.tokenize("""<unk> UNwanted , running"""   )
         self.assertListEqual(UpperCamelCase   ,     ["""<unk>""", """unwanted""", """,""", """running"""]   )

         self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase   )   ,     [0, 4, 8, 7]   )






   def 	_lowerCAmelCase   (		self       :		Any   )			->						int:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[Any]   =	TransfoXLTokenizer(lower_case=UpperCamelCase   )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """   )   ,     ["""hello""", """!""", """how""", """are""", """you""", """?"""]   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	TransfoXLTokenizer(lower_case=UpperCamelCase   )

         self.assertListEqual(
             tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """   )   ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]   )






   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Dict   =	TransfoXLTokenizer(lower_case=UpperCamelCase   )
         lowerCAmelCase__      :			Union[str, Any]   =	"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
         lowerCAmelCase__      :			Union[str, Any]   =	[
             """Hello""",
             """(""",
             """bracket""",
             """)""",
             """and""",
             """side""",
             """@-@""",
             """scrolled""",
             """[""",
             """and""",
             """]""",
             """Henry""",
             """'s""",
             """$""",
             """5""",
             """@,@""",
             """000""",
             """with""",
             """3""",
             """@.@""",
             """34""",
             """m""",
             """.""",
             """What""",
             """'s""",
             """up""",
             """!""",
             """?""",
         ]

         self.assertListEqual(tokenizer.tokenize(UpperCamelCase   )   ,     UpperCamelCase   )

         self.assertEqual(tokenizer.convert_tokens_to_string(UpperCamelCase   )   ,     UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			List[str]   =	self.get_tokenizer()
         lowerCAmelCase__      :			int   =	len(UpperCamelCase   )

         tokenizer.add_tokens(["""new1""", """new2"""]   )
         tokenizer.move_added_token("""new1"""   ,     1   )

         # Check that moved token is not copied (duplicate)
         self.assertEqual(len(UpperCamelCase   )   ,     original_len + 2   )
         # Check that token is moved to specified id
         self.assertEqual(tokenizer.encode("""new1"""   )   ,     [1]   )
         self.assertEqual(tokenizer.decode([1]   )   ,     """new1"""   )











"""simple docstring"""


from .configuration_bert_masked import MaskedBertConfig
from .modeling_bert_masked import (
    MaskedBertForMultipleChoice,
    MaskedBertForQuestionAnswering,
    MaskedBertForSequenceClassification,
    MaskedBertForTokenClassification,
    MaskedBertModel,
)
from .modules import *





