




import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
   from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
   lowercase__				:      Dict	   = get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
   from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowercase__				:      Any	   = 128_022
lowercase__				:      Tuple	   = 128_028




@require_sentencepiece
class       a__	( UpperCamelCase__       , unittest.TestCase				):
     a		:      Union[str, Any]		 =     MaMaaaTokenizer
     a		:      List[Any]		 =     False
     a		:      Union[str, Any]		 =     False
     a		:      Optional[Any]		 =     True







     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          super().setUp()

          a     =				["</s>", "<unk>", "▁This", "▁is", "▁a", "▁t", "est", "\u0120", "<pad>"]
          a     =				dict(zip(A  ,		range(len(A			)			)			)			)
          a     =				Path(self.tmpdirname			)
          save_json(A  ,		save_dir / VOCAB_FILES_NAMES["vocab_file"]			)
          if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
               copyfile(A  ,		save_dir / VOCAB_FILES_NAMES["spm_file"]			)

          a     =				MaMaaaTokenizer.from_pretrained(self.tmpdirname			)
          tokenizer.save_pretrained(self.tmpdirname			)







     def       lowerCAmelCase_    (     self  ,		**A			)  ->   str:



          '''simple docstring'''



          return MaMaaaTokenizer.from_pretrained(self.tmpdirname  ,		**A			)







     def       lowerCAmelCase_    (     self  ,		A			)  ->   int:



          '''simple docstring'''



          return (
              "This is a test",
              "This is a test",
          )







     def       lowerCAmelCase_    (     self			)  ->   Tuple:



          '''simple docstring'''



          a     =				"</s>"
          a     =				0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(A			)  ,		A			)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(A			)  ,		A			)







     def       lowerCAmelCase_    (     self			)  ->   Union[str, Any]:



          '''simple docstring'''



          a     =				self.get_tokenizer()
          a     =				list(tokenizer.get_vocab().keys()			)

          self.assertEqual(vocab_keys[0]  ,		"</s>"			)
          self.assertEqual(vocab_keys[1]  ,		"<unk>"			)
          self.assertEqual(vocab_keys[-1]  ,		"<s>"			)
          self.assertEqual(len(A			)  ,		tokenizer.vocab_size + len(tokenizer.get_added_vocab()			)			)







     @unittest.skip("Skip this test while all models are still to be uploaded."			)
     def       lowerCAmelCase_    (     self			)  ->   List[str]:



          '''simple docstring'''



          pass







     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          a     =				self.get_tokenizer()

          a     =				tokenizer.tokenize("This is a test"			)
          self.assertListEqual(A  ,		["▁This", "▁is", "▁a", "▁t", "est"]			)

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(A			)  ,		[2, 3, 4, 5, 6]  ,		)

          a     =				tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]			)
          self.assertListEqual(A  ,		["▁This", "▁is", "▁a", "▁t", "est"]			)

          a     =				tokenizer.convert_tokens_to_string(A			)
          self.assertEqual(A  ,		"This is a test"			)







     @slow
     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          a     =				{"input_ids": [[128022, 110108, 397, 11, 38272, 2247, 124811, 285, 18105, 1586, 207, 7, 39534, 4428, 397, 1019, 18105, 1586, 207, 7, 41337, 16786, 241, 7, 20214, 17, 125690, 10398, 7, 44378, 58069, 68342, 7798, 7343, 11, 299, 33310, 4, 158, 37350, 94077, 4569, 299, 33310, 90, 4, 52840, 290, 4, 31270, 112, 299, 682, 4, 52840, 39953, 14079, 193, 52519, 90894, 17894, 120697, 11, 40445, 551, 17, 1019, 52519, 90894, 17756, 963, 11, 40445, 480, 17, 9792, 1120, 5173, 1393, 6240, 16786, 241, 120996, 28, 1245, 1393, 118240, 11123, 1019, 93612, 2691, 10618, 98058, 120409, 1928, 279, 4, 40683, 367, 178, 207, 1019, 103, 103121, 506, 65296, 5, 2], [128022, 21217, 367, 117, 125450, 128, 719, 7, 7308, 40, 93612, 12669, 1116, 16704, 71, 17785, 3699, 15592, 35, 144, 9584, 241, 11943, 713, 950, 799, 2247, 88427, 150, 149, 118813, 120706, 1019, 106906, 81518, 28, 1224, 22799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128022, 1658, 123311, 5155, 5578, 4722, 279, 14947, 2366, 1120, 1197, 14, 1348, 9232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=A  ,		model_name="facebook/m2m100_418M"  ,		revision="c168bae485c864188cf9aa0e4108b0b6934dc91e"  ,		)







@require_torch
@require_sentencepiece
@require_tokenizers
class       a__	( unittest.TestCase				):
     a		:      Union[str, Any]		 =     """facebook/m2m100_418M"""
     a		:      Tuple		 =     [
         """In my opinion, there are two levels of response from the French government.""",
         """NSA Affair Emphasizes Complete Lack of Debate on Intelligence""",
     ]
     a		:      str		 =     [
         """Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.""",
         """L'affaire NSA souligne l'absence totale de débat sur le renseignement""",
     ]

     # fmt: off
     a		:      Any		 =     [EN_CODE, 593, 1949, 11_5781, 4, 7_1586, 4234, 6_0633, 12_6233, 432, 12_3808, 1_5592, 1197, 11_7132, 12_0618, 5, 2]







     @classmethod
     def       lowerCAmelCase_    (     cls			)  ->   Any:



          '''simple docstring'''



          a     =				MaMaaaTokenizer.from_pretrained(
              cls.checkpoint_name  ,		src_lang="en"  ,		tgt_lang="fr"			)
          a     =				1
          return cls







     def       lowerCAmelCase_    (     self			)  ->   Dict:



          '''simple docstring'''



          self.assertEqual(self.tokenizer.get_lang_id("ar"			)  ,		128006			)
          self.assertEqual(self.tokenizer.get_lang_id("en"			)  ,		128022			)
          self.assertEqual(self.tokenizer.get_lang_id("ro"			)  ,		128076			)
          self.assertEqual(self.tokenizer.get_lang_id("mr"			)  ,		128063			)







     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          a     =				self.tokenizer.get_vocab()
          self.assertEqual(len(A			)  ,		self.tokenizer.vocab_size			)
          self.assertEqual(vocab["<unk>"]  ,		3			)
          self.assertIn(self.tokenizer.get_lang_token("en"			)  ,		A			)







     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          a     =				"en"
          a     =				self.tokenizer.batch_encode_plus(self.src_text			).input_ids[0]
          self.assertListEqual(self.expected_src_tokens  ,		A			)







     def       lowerCAmelCase_    (     self			)  ->   int:



          '''simple docstring'''



          self.assertIn(A  ,		self.tokenizer.all_special_ids			)
          # fmt: off
          a     =				[FR_CODE, 5364, 82, 8642, 4, 294, 47, 8, 14028, 136, 3286, 9706, 6, 90797, 6, 144012, 162, 88128, 30061, 5, 2]
          # fmt: on
          a     =				self.tokenizer.decode(A  ,		skip_special_tokens=A			)
          a     =				self.tokenizer.decode(generated_ids[1:]  ,		skip_special_tokens=A			)
          self.assertEqual(A  ,		A			)
          self.assertNotIn(self.tokenizer.eos_token  ,		A			)







     def       lowerCAmelCase_    (     self			)  ->   Tuple:



          '''simple docstring'''



          a     =				tempfile.mkdtemp()
          a     =				self.tokenizer.lang_token_to_id
          self.tokenizer.save_pretrained(A			)
          a     =				MaMaaaTokenizer.from_pretrained(A			)
          self.assertDictEqual(new_tok.lang_token_to_id  ,		A			)







     @require_torch
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          a     =				"en"
          a     =				"fr"

          a     =				self.tokenizer(self.src_text  ,		text_target=self.tgt_text  ,		padding=A  ,		return_tensors="pt"			)

          a     =				shift_tokens_right(
              batch["labels"]  ,		self.tokenizer.pad_token_id  ,		self.tokenizer.eos_token_id			)

          for k in batch:
               a     =				batch[k].tolist()
          # batch = {k: v.tolist() for k,v in batch.items()}
          # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
          # batch.decoder_inputs_ids[0][0] ==
          assert batch.input_ids[1][0] == EN_CODE
          assert batch.input_ids[1][-1] == 2
          assert batch.labels[1][0] == FR_CODE
          assert batch.labels[1][-1] == 2
          assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]







     @require_torch
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          a     =				"mr"
          self.assertListEqual(self.tokenizer.prefix_tokens  ,		[self.tokenizer.get_lang_id("mr"			)]			)
          self.assertListEqual(self.tokenizer.suffix_tokens  ,		[self.tokenizer.eos_token_id]			)

          a     =				"zh"
          self.assertListEqual(self.tokenizer.prefix_tokens  ,		[self.tokenizer.get_lang_id("zh"			)]			)
          self.assertListEqual(self.tokenizer.suffix_tokens  ,		[self.tokenizer.eos_token_id]			)







     @require_torch
     def       lowerCAmelCase_    (     self			)  ->   Optional[int]:



          '''simple docstring'''



          a     =				"mr"
          self.tokenizer._switch_to_target_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens  ,		[self.tokenizer.get_lang_id("mr"			)]			)
          self.assertListEqual(self.tokenizer.suffix_tokens  ,		[self.tokenizer.eos_token_id]			)
          self.tokenizer._switch_to_input_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens  ,		[self.tokenizer.get_lang_id(self.tokenizer.src_lang			)]			)

          a     =				"zh"
          self.tokenizer._switch_to_target_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens  ,		[self.tokenizer.get_lang_id("zh"			)]			)
          self.assertListEqual(self.tokenizer.suffix_tokens  ,		[self.tokenizer.eos_token_id]			)
          self.tokenizer._switch_to_input_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens  ,		[self.tokenizer.get_lang_id(self.tokenizer.src_lang			)]			)







     @require_torch
     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          a     =				self.tokenizer._build_translation_inputs("A test"  ,		return_tensors="pt"  ,		src_lang="en"  ,		tgt_lang="ar"			)

          self.assertEqual(
              nested_simplify(A			)  ,		{
                  # en_XX, A, test, EOS
                  "input_ids": [[128022, 58, 4183, 2]],
                  "attention_mask": [[1, 1, 1, 1]],
                  # ar_AR
                  "forced_bos_token_id": 128006,
              }  ,		)







from manim import *




class       a__	( UpperCamelCase__				):







     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          a     =				Rectangle(height=0.5  ,		width=0.5			)
          a     =				Rectangle(height=0.4_6  ,		width=0.4_6			).set_stroke(width=0			)

          a     =				Rectangle(height=0.2_5  ,		width=0.2_5			)

          a     =				[mem.copy() for i in range(6			)]
          a     =				[mem.copy() for i in range(6			)]
          a     =				VGroup(*A			).arrange(A  ,		buff=0			)
          a     =				VGroup(*A			).arrange(A  ,		buff=0			)
          a     =				VGroup(A  ,		A			).arrange(A  ,		buff=0			)
          a     =				Text("CPU"  ,		font_size=24			)
          a     =				Group(A  ,		A			).arrange(A  ,		buff=0.5  ,		aligned_edge=A			)
          cpu.move_to([-2.5, -0.5, 0]			)
          self.add(A			)

          a     =				[mem.copy() for i in range(4			)]
          a     =				VGroup(*A			).arrange(A  ,		buff=0			)
          a     =				Text("GPU"  ,		font_size=24			)
          a     =				Group(A  ,		A			).arrange(A  ,		buff=0.5  ,		aligned_edge=A			)
          gpu.move_to([-1, -1, 0]			)
          self.add(A			)

          a     =				[mem.copy() for i in range(6			)]
          a     =				VGroup(*A			).arrange(A  ,		buff=0			)

          a     =				Text("Model"  ,		font_size=24			)
          a     =				Group(A  ,		A			).arrange(A  ,		buff=0.5  ,		aligned_edge=A			)
          model.move_to([3, -1.0, 0]			)
          self.add(A			)

          a     =				[]
          a     =				[]

          for i, rect in enumerate(A			):
               a     =				fill.copy().set_fill(A  ,		opacity=0.8			)
               target.move_to(A			)
               model_arr.append(A			)

               a     =				Rectangle(height=0.4_6  ,		width=0.4_6			).set_stroke(width=0.0			).set_fill(A  ,		opacity=0.8			)
               cpu_target.move_to(cpu_left_col_base[i]			)
               model_cpu_arr.append(A			)

          self.add(*A  ,		*A			)

          a     =				[meta_mem.copy() for i in range(6			)]
          a     =				[meta_mem.copy() for i in range(6			)]
          a     =				VGroup(*A			).arrange(A  ,		buff=0			)
          a     =				VGroup(*A			).arrange(A  ,		buff=0			)
          a     =				VGroup(A  ,		A			).arrange(A  ,		buff=0			)
          a     =				Text("Disk"  ,		font_size=24			)
          a     =				Group(A  ,		A			).arrange(A  ,		buff=0.5  ,		aligned_edge=A			)
          disk.move_to([-4, -1.2_5, 0]			)
          self.add(A  ,		A			)

          a     =				Square(side_length=2.2			)
          key.move_to([-5, 2, 0]			)

          a     =				MarkupText(
              F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''  ,		font_size=18  ,		)

          key_text.move_to([-5, 2.4, 0]			)

          self.add(A  ,		A			)

          a     =				MarkupText(
              F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''  ,		font_size=18  ,		)

          blue_text.next_to(A  ,		DOWN * 2.4  ,		aligned_edge=key_text.get_left()			)
          self.add(A			)

          a     =				MarkupText(
              F'''Now watch as an input is passed through the model\nand how the memory is utilized and handled.'''  ,		font_size=24  ,		)
          step_a.move_to([2, 2, 0]			)

          self.play(Write(A			)			)

          a     =				Square(0.3			)
          input.set_fill(A  ,		opacity=1.0			)
          input.set_stroke(width=0.0			)
          input.next_to(model_base[0]  ,		A  ,		buff=0.5			)

          self.play(Write(A			)			)

          input.generate_target()
          input.target.next_to(model_arr[0]  ,		direction=A  ,		buff=0.0_2			)
          self.play(MoveToTarget(A			)			)

          self.play(FadeOut(A			)			)

          a     =				Arrow(start=A  ,		end=A  ,		color=A  ,		buff=0.5			)
          a.next_to(model_arr[0].get_left()  ,		A  ,		buff=0.2			)

          model_cpu_arr[0].generate_target()
          model_cpu_arr[0].target.move_to(gpu_rect[0]			)

          a     =				MarkupText(
              F'''As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.'''  ,		font_size=24  ,		)
          step_a.move_to([2, 2, 0]			)

          self.play(Write(A  ,		run_time=3			)			)

          a     =				{"run_time": 1, "fade_in": True, "fade_out": True, "buff": 0.0_2}

          self.play(
              Write(A			)  ,		Circumscribe(model_arr[0]  ,		color=A  ,		**A			)  ,		Circumscribe(model_cpu_arr[0]  ,		color=A  ,		**A			)  ,		Circumscribe(gpu_rect[0]  ,		color=A  ,		**A			)  ,		)
          self.play(MoveToTarget(model_cpu_arr[0]			)			)

          a     =				a.copy()
          for i in range(6			):
               a_c.next_to(model_arr[i].get_right() + 0.0_2  ,		A  ,		buff=0.2			)

               input.generate_target()
               input.target.move_to(model_arr[i].get_right() + 0.0_2			)

               a     =				AnimationGroup(
                   FadeOut(A  ,		run_time=0.5			)  ,		MoveToTarget(A  ,		run_time=0.5			)  ,		FadeIn(A  ,		run_time=0.5			)  ,		lag_ratio=0.2			)

               self.play(A			)

               model_cpu_arr[i].generate_target()
               model_cpu_arr[i].target.move_to(cpu_left_col_base[i]			)

               if i < 5:
                    model_cpu_arr[i + 1].generate_target()
                    model_cpu_arr[i + 1].target.move_to(gpu_rect[0]			)
                    if i >= 1:
                         a     =				0.7

                    self.play(
                        Circumscribe(model_arr[i]  ,		**A			)  ,		Circumscribe(cpu_left_col_base[i]  ,		**A			)  ,		Circumscribe(cpu_left_col_base[i + 1]  ,		color=A  ,		**A			)  ,		Circumscribe(gpu_rect[0]  ,		color=A  ,		**A			)  ,		Circumscribe(model_arr[i + 1]  ,		color=A  ,		**A			)  ,		)
                    if i < 1:
                         self.play(
                             MoveToTarget(model_cpu_arr[i]			)  ,		MoveToTarget(model_cpu_arr[i + 1]			)  ,		)
                    else:
                         self.play(
                             MoveToTarget(model_cpu_arr[i]  ,		run_time=0.7			)  ,		MoveToTarget(model_cpu_arr[i + 1]  ,		run_time=0.7			)  ,		)
               else:
                    model_cpu_arr[i].generate_target()
                    model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]			)
                    input.generate_target()
                    input.target.next_to(model_arr[-1].get_right()  ,		RIGHT + 0.0_2  ,		buff=0.2			)

                    self.play(
                        Circumscribe(model_arr[-1]  ,		color=A  ,		**A			)  ,		Circumscribe(cpu_left_col_base[-1]  ,		color=A  ,		**A			)  ,		Circumscribe(gpu_rect[0]  ,		color=A  ,		**A			)  ,		)

                    self.play(MoveToTarget(model_cpu_arr[i]			)			)

               a     =				a_c
               a     =				a_c.copy()

          input.generate_target()
          input.target.next_to(model_base[-1]  ,		RIGHT + 0.0_2  ,		buff=0.5			)
          self.play(
              FadeOut(A			)  ,		FadeOut(A  ,		run_time=0.5			)  ,		)

          a     =				MarkupText(F'''Inference on a model too large for GPU memory\nis successfully completed.'''  ,		font_size=24			)
          step_a.move_to([2, 2, 0]			)

          self.play(Write(A  ,		run_time=3			)  ,		MoveToTarget(A			)			)

          self.wait()

