





import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_           =      get_tests_dir('fixtures/test_sentencepiece.model')

@require_sentencepiece
@require_tokenizers
class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase						,				unittest.TestCase  ):
      A_						:    Any							=   ReformerTokenizer
      A_						:    Optional[Any]							=   ReformerTokenizerFast
      A_						:    str							=   True
      A_						:    Dict							=   False
      A_						:    Tuple							=   True


      def        a			(self	:				Union[str, Any]				):





         """simple docstring"""



         super().setUp()

         __snake_case								=       ReformerTokenizer(a__    ,      keep_accents=a__				)
         tokenizer.save_pretrained(self.tmpdirname				)


      def        a			(self	:				Optional[int]				):





         """simple docstring"""



         __snake_case								=       '''<s>'''
         __snake_case								=       1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__				)    ,      a__				)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__				)    ,      a__				)


      def        a			(self	:				List[Any]				):





         """simple docstring"""



         __snake_case								=       list(self.get_tokenizer().get_vocab().keys()				)

         self.assertEqual(vocab_keys[0]    ,      '''<unk>'''				)
         self.assertEqual(vocab_keys[1]    ,      '''<s>'''				)
         self.assertEqual(vocab_keys[-1]    ,      '''j'''				)
         self.assertEqual(len(a__				)    ,      1000				)


      def        a			(self	:				List[Any]				):





         """simple docstring"""



         self.assertEqual(self.get_tokenizer().vocab_size    ,      1000				)


      def        a			(self	:				str				):





         """simple docstring"""



         if not self.test_rust_tokenizer:
            return

         __snake_case								=       self.get_tokenizer()
         __snake_case								=       self.get_rust_tokenizer()

         __snake_case								=       '''I was born in 92000, and this is falsé.'''

         __snake_case								=       tokenizer.tokenize(a__				)
         __snake_case								=       rust_tokenizer.tokenize(a__				)
         self.assertListEqual(a__    ,      a__				)

         __snake_case								=       tokenizer.encode(a__    ,      add_special_tokens=a__				)
         __snake_case								=       rust_tokenizer.encode(a__    ,      add_special_tokens=a__				)
         self.assertListEqual(a__    ,      a__				)

         __snake_case								=       self.get_rust_tokenizer()
         __snake_case								=       tokenizer.encode(a__				)
         __snake_case								=       rust_tokenizer.encode(a__				)
         self.assertListEqual(a__    ,      a__				)


      def        a			(self	:				str    ,      a__	:				Any=15				):





         """simple docstring"""



         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
               __snake_case								=       self.rust_tokenizer_class.from_pretrained(a__    ,      **a__				)

               # Simple input
               __snake_case								=       '''This is a simple input'''
               __snake_case								=       ['''This is a simple input 1''', '''This is a simple input 2''']
               __snake_case								=       ('''This is a simple input''', '''This is a pair''')
               __snake_case								=       [
                   ('''This is a simple input 1''', '''This is a simple input 2'''),
                   ('''This is a simple pair 1''', '''This is a simple pair 2'''),
               ]

               # Simple input tests
               self.assertRaises(a__    ,      tokenizer_r.encode    ,      a__    ,      max_length=a__    ,      padding='''max_length'''				)

               # Simple input
               self.assertRaises(a__    ,      tokenizer_r.encode_plus    ,      a__    ,      max_length=a__    ,      padding='''max_length'''				)

               # Simple input
               self.assertRaises(
                   a__    ,      tokenizer_r.batch_encode_plus    ,      a__    ,      max_length=a__    ,      padding='''max_length'''    ,      )

               # Pair input
               self.assertRaises(a__    ,      tokenizer_r.encode    ,      a__    ,      max_length=a__    ,      padding='''max_length'''				)

               # Pair input
               self.assertRaises(a__    ,      tokenizer_r.encode_plus    ,      a__    ,      max_length=a__    ,      padding='''max_length'''				)

               # Pair input
               self.assertRaises(
                   a__    ,      tokenizer_r.batch_encode_plus    ,      a__    ,      max_length=a__    ,      padding='''max_length'''    ,      )


      def        a			(self	:				Optional[int]				):





         """simple docstring"""



         pass


      def        a			(self	:				Tuple				):





         """simple docstring"""



         __snake_case								=       ReformerTokenizer(a__    ,      keep_accents=a__				)

         __snake_case								=       tokenizer.tokenize('''This is a test'''				)
         self.assertListEqual(a__    ,      ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(a__				)    ,      [285, 46, 10, 170, 382]    ,      )

         __snake_case								=       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
         self.assertListEqual(
             a__    ,      [
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''9''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''é''',
                 '''.''',
             ]    ,      )
         __snake_case								=       tokenizer.convert_tokens_to_ids(a__				)
         self.assertListEqual(
             a__    ,      [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]    ,      )

         __snake_case								=       tokenizer.convert_ids_to_tokens(a__				)
         self.assertListEqual(
             a__    ,      [
                 SPIECE_UNDERLINE + '''I''',
                 SPIECE_UNDERLINE + '''was''',
                 SPIECE_UNDERLINE + '''b''',
                 '''or''',
                 '''n''',
                 SPIECE_UNDERLINE + '''in''',
                 SPIECE_UNDERLINE + '''''',
                 '''<unk>''',
                 '''2''',
                 '''0''',
                 '''0''',
                 '''0''',
                 ''',''',
                 SPIECE_UNDERLINE + '''and''',
                 SPIECE_UNDERLINE + '''this''',
                 SPIECE_UNDERLINE + '''is''',
                 SPIECE_UNDERLINE + '''f''',
                 '''al''',
                 '''s''',
                 '''<unk>''',
                 '''.''',
             ]    ,      )


      @cached_property
      def        a			(self	:				Tuple				):





         """simple docstring"""



         return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''				)


      @slow
      def        a			(self	:				Optional[int]				):





         """simple docstring"""



         __snake_case								=       '''Hello World!'''
         __snake_case								=       [126, 32, 262, 152, 38, 72, 287]

         self.assertListEqual(a__    ,      self.big_tokenizer.encode(a__				)				)


      @slow
      def        a			(self	:				List[str]				):





         """simple docstring"""



         __snake_case								=       (
             '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
             ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
         )
         __snake_case								=       [
             108,
             265,
             24,
             111,
             4,
             258,
             156,
             35,
             28,
             275,
             3,
             259,
             297,
             260,
             84,
             4,
             35,
             110,
             44,
             8,
             259,
             91,
             268,
             21,
             11,
             209,
             274,
             109,
             266,
             277,
             117,
             86,
             93,
             315,
             258,
             278,
             258,
             277,
             258,
             0,
             258,
             288,
             258,
             319,
             258,
             0,
             258,
             0,
             258,
             0,
             258,
             0,
             258,
             287,
             258,
             315,
             258,
             289,
             258,
             278,
             99,
             269,
             266,
             262,
             8,
             259,
             241,
             4,
             217,
             230,
             268,
             266,
             55,
             168,
             106,
             75,
             193,
             266,
             223,
             27,
             49,
             26,
             282,
             25,
             264,
             299,
             19,
             26,
             0,
             258,
             277,
             117,
             86,
             93,
             176,
             183,
             270,
             11,
             262,
             42,
             61,
             265,
         ]

         self.assertListEqual(a__    ,      self.big_tokenizer.encode(a__				)				)


      @require_torch
      @slow
      def        a			(self	:				List[Any]				):





         """simple docstring"""



         import torch

         from transformers import ReformerConfig, ReformerModel

         # Build sequence
         __snake_case								=       list(self.big_tokenizer.get_vocab().keys()				)[:10]
         __snake_case								=       ''' '''.join(a__				)
         __snake_case								=       self.big_tokenizer.encode_plus(a__    ,      return_tensors='''pt'''				)
         __snake_case								=       self.big_tokenizer.batch_encode_plus([sequence, sequence]    ,      return_tensors='''pt'''				)

         __snake_case								=       ReformerConfig()
         # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
         __snake_case								=       encoded_sequence['''input_ids'''].shape
         __snake_case								=       ReformerModel(a__				)

         # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
         assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

         with torch.no_grad():
            model(**a__				)
            model(**a__				)


      @slow
      def        a			(self	:				List[Any]				):





         """simple docstring"""



         __snake_case								=       {'''input_ids''': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         # This tokenizer does not know some characters like ")".
         # That is the reason why we use very simple texts here.
         # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
         __snake_case								=       [
             '''This is a very simple sentence.''',
             '''The quick brown fox jumps over the lazy dog.''',
         ]

         self.tokenizer_integration_test_util(
             expected_encoding=a__    ,      model_name='''google/reformer-crime-and-punishment'''    ,      revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a'''    ,      padding=a__    ,      sequences=a__    ,      )








import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin

class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase						,				unittest.TestCase  ):
      A_						:    int							=   XLMTokenizer
      A_						:    Optional[Any]							=   False


      def        a			(self	:				Optional[int]				):





         """simple docstring"""



         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         __snake_case								=       [
             '''l''',
             '''o''',
             '''w''',
             '''e''',
             '''r''',
             '''s''',
             '''t''',
             '''i''',
             '''d''',
             '''n''',
             '''w</w>''',
             '''r</w>''',
             '''t</w>''',
             '''lo''',
             '''low''',
             '''er</w>''',
             '''low</w>''',
             '''lowest</w>''',
             '''newer</w>''',
             '''wider</w>''',
             '''<unk>''',
         ]
         __snake_case								=       dict(zip(a__    ,      range(len(a__				)				)				)				)
         __snake_case								=       ['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

         __snake_case								=       os.path.join(self.tmpdirname    ,      VOCAB_FILES_NAMES['''vocab_file''']				)
         __snake_case								=       os.path.join(self.tmpdirname    ,      VOCAB_FILES_NAMES['''merges_file''']				)
         with open(self.vocab_file    ,      '''w'''				) as fp:
            fp.write(json.dumps(a__				)				)
         with open(self.merges_file    ,      '''w'''				) as fp:
            fp.write('''\n'''.join(a__				)				)


      def        a			(self	:				int    ,      a__	:				Any				):





         """simple docstring"""



         __snake_case								=       '''lower newer'''
         __snake_case								=       '''lower newer'''
         return input_text, output_text


      def        a			(self	:				Optional[Any]				):





         """simple docstring"""



         __snake_case								=       XLMTokenizer(self.vocab_file    ,      self.merges_file				)

         __snake_case								=       '''lower'''
         __snake_case								=       ['''low''', '''er</w>''']
         __snake_case								=       tokenizer.tokenize(a__				)
         self.assertListEqual(a__    ,      a__				)

         __snake_case								=       tokens + ['''<unk>''']
         __snake_case								=       [14, 15, 20]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a__				)    ,      a__				)


      @slow
      def        a			(self	:				Union[str, Any]				):





         """simple docstring"""



         __snake_case								=       XLMTokenizer.from_pretrained('''xlm-mlm-en-2048'''				)

         __snake_case								=       tokenizer.encode('''sequence builders'''    ,      add_special_tokens=a__				)
         __snake_case								=       tokenizer.encode('''multi-sequence build'''    ,      add_special_tokens=a__				)

         __snake_case								=       tokenizer.build_inputs_with_special_tokens(a__				)
         __snake_case								=       tokenizer.build_inputs_with_special_tokens(a__    ,      a__				)

         assert encoded_sentence == [0] + text + [1]
         assert encoded_pair == [0] + text + [1] + text_a + [1]

