





'''simple docstring'''






import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


A__    :       Union[str, Any]              =get_tests_dir('''fixtures/test_sentencepiece.model''')




@require_sentencepiece
@require_tokenizers
class 			UpperCAmelCase					(      snake_case_							,		unittest.TestCase       ):
      _lowercase:      Any											=       ReformerTokenizer
      _lowercase:      List[str]											=       ReformerTokenizerFast
      _lowercase:      Optional[Any]											=       True
      _lowercase:      Tuple											=       False
      _lowercase:      Any											=       True




      def     lowercase__   (      self      :			int  )    ->				Optional[Any]:
            super().setUp()

            _lowerCAmelCase												=							ReformerTokenizer(__snake_case      ,      keep_accents=__snake_case  )
            tokenizer.save_pretrained(self.tmpdirname  )




      def     lowercase__   (      self      :			List[str]  )    ->				List[Any]:
            _lowerCAmelCase												=							"""<s>"""
            _lowerCAmelCase												=							1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case  )      ,      __snake_case  )
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case  )      ,      __snake_case  )




      def     lowercase__   (      self      :			int  )    ->				Tuple:
            _lowerCAmelCase												=							list(self.get_tokenizer().get_vocab().keys()  )

            self.assertEqual(vocab_keys[0]      ,      """<unk>"""  )
            self.assertEqual(vocab_keys[1]      ,      """<s>"""  )
            self.assertEqual(vocab_keys[-1]      ,      """j"""  )
            self.assertEqual(len(__snake_case  )      ,      10_00  )




      def     lowercase__   (      self      :			List[Any]  )    ->				int:
            self.assertEqual(self.get_tokenizer().vocab_size      ,      10_00  )




      def     lowercase__   (      self      :			str  )    ->				Dict:
            if not self.test_rust_tokenizer:
                  return

            _lowerCAmelCase												=							self.get_tokenizer()
            _lowerCAmelCase												=							self.get_rust_tokenizer()

            _lowerCAmelCase												=							"""I was born in 92000, and this is falsé."""

            _lowerCAmelCase												=							tokenizer.tokenize(__snake_case  )
            _lowerCAmelCase												=							rust_tokenizer.tokenize(__snake_case  )
            self.assertListEqual(__snake_case      ,      __snake_case  )

            _lowerCAmelCase												=							tokenizer.encode(__snake_case      ,      add_special_tokens=__snake_case  )
            _lowerCAmelCase												=							rust_tokenizer.encode(__snake_case      ,      add_special_tokens=__snake_case  )
            self.assertListEqual(__snake_case      ,      __snake_case  )

            _lowerCAmelCase												=							self.get_rust_tokenizer()
            _lowerCAmelCase												=							tokenizer.encode(__snake_case  )
            _lowerCAmelCase												=							rust_tokenizer.encode(__snake_case  )
            self.assertListEqual(__snake_case      ,      __snake_case  )




      def     lowercase__   (      self      :			int      ,      __snake_case      :			int=15  )    ->				Tuple:
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"  ):
                        _lowerCAmelCase												=							self.rust_tokenizer_class.from_pretrained(__snake_case      ,      **__snake_case  )

                        # Simple input
                        _lowerCAmelCase												=							"""This is a simple input"""
                        _lowerCAmelCase												=							["""This is a simple input 1""", """This is a simple input 2"""]
                        _lowerCAmelCase												=							("""This is a simple input""", """This is a pair""")
                        _lowerCAmelCase												=							[
                            ("""This is a simple input 1""", """This is a simple input 2"""),
                            ("""This is a simple pair 1""", """This is a simple pair 2"""),
                        ]

                        # Simple input tests
                        self.assertRaises(__snake_case      ,      tokenizer_r.encode      ,      __snake_case      ,      max_length=__snake_case      ,      padding="""max_length"""  )

                        # Simple input
                        self.assertRaises(__snake_case      ,      tokenizer_r.encode_plus      ,      __snake_case      ,      max_length=__snake_case      ,      padding="""max_length"""  )

                        # Simple input
                        self.assertRaises(
                            __snake_case      ,      tokenizer_r.batch_encode_plus      ,      __snake_case      ,      max_length=__snake_case      ,      padding="""max_length"""      ,      )

                        # Pair input
                        self.assertRaises(__snake_case      ,      tokenizer_r.encode      ,      __snake_case      ,      max_length=__snake_case      ,      padding="""max_length"""  )

                        # Pair input
                        self.assertRaises(__snake_case      ,      tokenizer_r.encode_plus      ,      __snake_case      ,      max_length=__snake_case      ,      padding="""max_length"""  )

                        # Pair input
                        self.assertRaises(
                            __snake_case      ,      tokenizer_r.batch_encode_plus      ,      __snake_case      ,      max_length=__snake_case      ,      padding="""max_length"""      ,      )




      def     lowercase__   (      self      :			Tuple  )    ->				Any:
            pass




      def     lowercase__   (      self      :			List[Any]  )    ->				Union[str, Any]:
            _lowerCAmelCase												=							ReformerTokenizer(__snake_case      ,      keep_accents=__snake_case  )

            _lowerCAmelCase												=							tokenizer.tokenize("""This is a test"""  )
            self.assertListEqual(__snake_case      ,      ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(__snake_case  )      ,      [2_85, 46, 10, 1_70, 3_82]      ,      )

            _lowerCAmelCase												=							tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
            self.assertListEqual(
                __snake_case      ,      [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ]      ,      )
            _lowerCAmelCase												=							tokenizer.convert_tokens_to_ids(__snake_case  )
            self.assertListEqual(
                __snake_case      ,      [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]      ,      )

            _lowerCAmelCase												=							tokenizer.convert_ids_to_tokens(__snake_case  )
            self.assertListEqual(
                __snake_case      ,      [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ]      ,      )




      @cached_property
      def     lowercase__   (      self      :			Any  )    ->				int:
            return ReformerTokenizer.from_pretrained("""google/reformer-crime-and-punishment"""  )




      @slow
      def     lowercase__   (      self      :			Tuple  )    ->				Dict:
            _lowerCAmelCase												=							"""Hello World!"""
            _lowerCAmelCase												=							[1_26, 32, 2_62, 1_52, 38, 72, 2_87]

            self.assertListEqual(__snake_case      ,      self.big_tokenizer.encode(__snake_case  )  )




      @slow
      def     lowercase__   (      self      :			List[str]  )    ->				Optional[int]:
            _lowerCAmelCase												=							(
                """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
                """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
            )
            _lowerCAmelCase												=							[
                1_08,
                2_65,
                24,
                1_11,
                4,
                2_58,
                1_56,
                35,
                28,
                2_75,
                3,
                2_59,
                2_97,
                2_60,
                84,
                4,
                35,
                1_10,
                44,
                8,
                2_59,
                91,
                2_68,
                21,
                11,
                2_09,
                2_74,
                1_09,
                2_66,
                2_77,
                1_17,
                86,
                93,
                3_15,
                2_58,
                2_78,
                2_58,
                2_77,
                2_58,
                0,
                2_58,
                2_88,
                2_58,
                3_19,
                2_58,
                0,
                2_58,
                0,
                2_58,
                0,
                2_58,
                0,
                2_58,
                2_87,
                2_58,
                3_15,
                2_58,
                2_89,
                2_58,
                2_78,
                99,
                2_69,
                2_66,
                2_62,
                8,
                2_59,
                2_41,
                4,
                2_17,
                2_30,
                2_68,
                2_66,
                55,
                1_68,
                1_06,
                75,
                1_93,
                2_66,
                2_23,
                27,
                49,
                26,
                2_82,
                25,
                2_64,
                2_99,
                19,
                26,
                0,
                2_58,
                2_77,
                1_17,
                86,
                93,
                1_76,
                1_83,
                2_70,
                11,
                2_62,
                42,
                61,
                2_65,
            ]

            self.assertListEqual(__snake_case      ,      self.big_tokenizer.encode(__snake_case  )  )




      @require_torch
      @slow
      def     lowercase__   (      self      :			Any  )    ->				Any:
            import torch

            from transformers import ReformerConfig, ReformerModel

            # Build sequence
            _lowerCAmelCase												=							list(self.big_tokenizer.get_vocab().keys()  )[:10]
            _lowerCAmelCase												=							""" """.join(__snake_case  )
            _lowerCAmelCase												=							self.big_tokenizer.encode_plus(__snake_case      ,      return_tensors="""pt"""  )
            _lowerCAmelCase												=							self.big_tokenizer.batch_encode_plus([sequence, sequence]      ,      return_tensors="""pt"""  )

            _lowerCAmelCase												=							ReformerConfig()
            # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
            _lowerCAmelCase												=							encoded_sequence["""input_ids"""].shape
            _lowerCAmelCase												=							ReformerModel(__snake_case  )

            # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
            assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

            with torch.no_grad():
                  model(**__snake_case  )
                  model(**__snake_case  )




      @slow
      def     lowercase__   (      self      :			Union[str, Any]  )    ->				str:
            # fmt: off
            _lowerCAmelCase												=							{"""input_ids""": [[1_08, 2_65, 24, 1_11, 4, 2_58, 1_56, 7, 51, 2_79, 58, 7, 76, 25, 69, 2_78], [1_40, 2_43, 2_64, 1_34, 17, 2_67, 77, 2_63, 22, 2_62, 2_97, 2_58, 3_04, 1_77, 2_79, 2_66, 14, 89, 13, 35, 2_61, 2_99, 2_72, 1_37, 2_75, 2_78]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
            # fmt: on

            # This tokenizer does not know some characters like ")".
            # That is the reason why we use very simple texts here.
            # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
            _lowerCAmelCase												=							[
                """This is a very simple sentence.""",
                """The quick brown fox jumps over the lazy dog.""",
            ]

            self.tokenizer_integration_test_util(
                expected_encoding=__snake_case      ,      model_name="""google/reformer-crime-and-punishment"""      ,      revision="""0e6c3decb8211d49bf881013425dc8b0448b3f5a"""      ,      padding=__snake_case      ,      sequences=__snake_case      ,      )













'''simple docstring'''






import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_huggingface_suite,
    require_multi_gpu,
    require_single_gpu,
)
from accelerate.utils import patch_environment




@require_huggingface_suite
class 			UpperCAmelCase					(      unittest.TestCase       ):




      def     lowercase__   (      self      :			int  )    ->				Union[str, Any]:
            _lowerCAmelCase												=							inspect.getfile(accelerate.test_utils  )
            _lowerCAmelCase												=							os.path.sep.join(
                mod_file.split(os.path.sep  )[:-1] + ["""scripts""", """external_deps""", """test_metrics.py"""]  )

            from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401

            _lowerCAmelCase												=							test_metrics




      @require_cpu
      def     lowercase__   (      self      :			Optional[Any]  )    ->				Optional[Any]:
            debug_launcher(self.test_metrics.main      ,      num_processes=1  )




      @require_cpu
      def     lowercase__   (      self      :			Tuple  )    ->				Tuple:
            debug_launcher(self.test_metrics.main  )




      @require_single_gpu
      def     lowercase__   (      self      :			Union[str, Any]  )    ->				str:
            self.test_metrics.main()




      @require_multi_gpu
      def     lowercase__   (      self      :			str  )    ->				List[str]:
            print(f"Found {torch.cuda.device_count()} devices."  )
            _lowerCAmelCase												=							["""torchrun""", f"--nproc_per_node={torch.cuda.device_count()}", self.test_file_path]
            with patch_environment(omp_num_threads=1  ):
                  execute_subprocess_async(__snake_case      ,      env=os.environ.copy()  )






