




"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_A													=      {
    'configuration_mvp': ['MVP_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MvpConfig', 'MvpOnnxConfig'],
    'tokenization_mvp': ['MvpTokenizer'],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _A													=      ['MvpTokenizerFast']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _A													=      [
      'MVP_PRETRAINED_MODEL_ARCHIVE_LIST',
      'MvpForCausalLM',
      'MvpForConditionalGeneration',
      'MvpForQuestionAnswering',
      'MvpForSequenceClassification',
      'MvpModel',
      'MvpPreTrainedModel',
  ]

if TYPE_CHECKING:
  from .configuration_mvp import MVP_PRETRAINED_CONFIG_ARCHIVE_MAP, MvpConfig, MvpOnnxConfig
  from .tokenization_mvp import MvpTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_mvp_fast import MvpTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_mvp import (
        MVP_PRETRAINED_MODEL_ARCHIVE_LIST,
        MvpForCausalLM,
        MvpForConditionalGeneration,
        MvpForQuestionAnswering,
        MvpForSequenceClassification,
        MvpModel,
        MvpPreTrainedModel,
    )

else:
  import sys

  _A													=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)











"""simple docstring"""


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_A													=      get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_A													=      2_5_6_0_4_7
_A													=      2_5_6_1_4_5


@require_sentencepiece
@require_tokenizers
class _lowerCamelCase    (					a_       ,	unittest.TestCase						):
   _lowerCamelCase    :Any     				=     NllbTokenizer
   _lowerCamelCase    :Dict     				=     NllbTokenizerFast
   _lowerCamelCase    :str     				=     True
   _lowerCamelCase    :Optional[Any]     				=     True
   _lowerCamelCase    :Union[str, Any]     				=     {}






   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						int:

         """simple docstring"""
         super().setUp()

         # We have a SentencePiece fixture for testing
         lowerCAmelCase__      :			Optional[int]   =	NllbTokenizer(UpperCamelCase   ,     keep_accents=UpperCamelCase   )
         tokenizer.save_pretrained(self.tmpdirname   )






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						List[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Dict   =	NllbTokenizer(UpperCamelCase   ,     keep_accents=UpperCamelCase   )

         lowerCAmelCase__      :			List[str]   =	tokenizer.tokenize("""This is a test"""   )
         self.assertListEqual(UpperCamelCase   ,     ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]   )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(UpperCamelCase   )   ,     [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]   ,     )

         lowerCAmelCase__      :			Dict   =	tokenizer.tokenize("""I was born in 92000, and this is falsé."""   )
         self.assertListEqual(
             UpperCamelCase   ,     [
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """9""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """é""",
                 """.""",
             ]   ,     )
         lowerCAmelCase__      :			Optional[Any]   =	tokenizer.convert_tokens_to_ids(UpperCamelCase   )
         self.assertListEqual(
             UpperCamelCase   ,     [
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
             ]   ,     )

         lowerCAmelCase__      :			List[str]   =	tokenizer.convert_ids_to_tokens(UpperCamelCase   )
         self.assertListEqual(
             UpperCamelCase   ,     [
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """<unk>""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """<unk>""",
                 """.""",
             ]   ,     )






   def 	_lowerCAmelCase   (		self       :		Any   )			->						Any:

         """simple docstring"""
         lowerCAmelCase__      :			str   =	(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
                     lowerCAmelCase__      :			Union[str, Any]   =	self.rust_tokenizer_class.from_pretrained(UpperCamelCase   ,     **UpperCamelCase   )
                     lowerCAmelCase__      :			str   =	self.tokenizer_class.from_pretrained(UpperCamelCase   ,     **UpperCamelCase   )

                     lowerCAmelCase__      :			int   =	tempfile.mkdtemp()

                     lowerCAmelCase__      :			Tuple   =	tokenizer_r.save_pretrained(UpperCamelCase   )
                     lowerCAmelCase__      :			Optional[int]   =	tokenizer_p.save_pretrained(UpperCamelCase   )

                     # Checks it save with the same files + the tokenizer.json file for the fast one
                     self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files   )   )
                     lowerCAmelCase__      :			Dict   =	tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f   )
                     self.assertSequenceEqual(UpperCamelCase   ,     UpperCamelCase   )

                     # Checks everything loads correctly in the same way
                     lowerCAmelCase__      :			int   =	tokenizer_r.from_pretrained(UpperCamelCase   )
                     lowerCAmelCase__      :			Union[str, Any]   =	tokenizer_p.from_pretrained(UpperCamelCase   )

                     # Check special tokens are set accordingly on Rust and Python
                     for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(UpperCamelCase   ,     UpperCamelCase   )   )

                     shutil.rmtree(UpperCamelCase   )

                     # Save tokenizer rust, legacy_format=True
                     lowerCAmelCase__      :			List[str]   =	tempfile.mkdtemp()

                     lowerCAmelCase__      :			Optional[Any]   =	tokenizer_r.save_pretrained(UpperCamelCase   ,     legacy_format=UpperCamelCase   )
                     lowerCAmelCase__      :			List[str]   =	tokenizer_p.save_pretrained(UpperCamelCase   )

                     # Checks it save with the same files
                     self.assertSequenceEqual(UpperCamelCase   ,     UpperCamelCase   )

                     # Checks everything loads correctly in the same way
                     lowerCAmelCase__      :			List[str]   =	tokenizer_r.from_pretrained(UpperCamelCase   )
                     lowerCAmelCase__      :			Optional[Any]   =	tokenizer_p.from_pretrained(UpperCamelCase   )

                     # Check special tokens are set accordingly on Rust and Python
                     for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(UpperCamelCase   ,     UpperCamelCase   )   )

                     shutil.rmtree(UpperCamelCase   )

                     # Save tokenizer rust, legacy_format=False
                     lowerCAmelCase__      :			List[Any]   =	tempfile.mkdtemp()

                     lowerCAmelCase__      :			int   =	tokenizer_r.save_pretrained(UpperCamelCase   ,     legacy_format=UpperCamelCase   )
                     lowerCAmelCase__      :			str   =	tokenizer_p.save_pretrained(UpperCamelCase   )

                     # Checks it saved the tokenizer.json file
                     self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files   )   )

                     # Checks everything loads correctly in the same way
                     lowerCAmelCase__      :			Dict   =	tokenizer_r.from_pretrained(UpperCamelCase   )
                     lowerCAmelCase__      :			Optional[int]   =	tokenizer_p.from_pretrained(UpperCamelCase   )

                     # Check special tokens are set accordingly on Rust and Python
                     for key in tokenizer_pp.special_tokens_map:
                           self.assertTrue(hasattr(UpperCamelCase   ,     UpperCamelCase   )   )

                     shutil.rmtree(UpperCamelCase   )






   @require_torch
   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						List[str]:

         """simple docstring"""
         if not self.test_seqaseq:
               return

         lowerCAmelCase__      :			int   =	self.get_tokenizers()
         for tokenizer in tokenizers:
               with self.subTest(f"""{tokenizer.__class__.__name__}"""   ):
                     # Longer text that will definitely require truncation.
                     lowerCAmelCase__      :			Any   =	[
                         """ UN Chief Says There Is No Military Solution in Syria""",
                         """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
                         """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
                         """ will only worsen the violence and misery for millions of people.""",
                     ]
                     lowerCAmelCase__      :			Optional[int]   =	[
                         """Şeful ONU declară că nu există o soluţie militară în Siria""",
                         """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
                         """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
                         """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
                     ]
                     try:
                           lowerCAmelCase__      :			Dict   =	tokenizer.prepare_seqaseq_batch(
                               src_texts=UpperCamelCase   ,     tgt_texts=UpperCamelCase   ,     max_length=3   ,     max_target_length=10   ,     return_tensors="""pt"""   ,     src_lang="""eng_Latn"""   ,     tgt_lang="""ron_Latn"""   ,     )
                     except NotImplementedError:
                           return
                     self.assertEqual(batch.input_ids.shape[1]   ,     3   )
                     self.assertEqual(batch.labels.shape[1]   ,     10   )
                     # max_target_length will default to max_length if not specified
                     lowerCAmelCase__      :			str   =	tokenizer.prepare_seqaseq_batch(
                         UpperCamelCase   ,     tgt_texts=UpperCamelCase   ,     max_length=3   ,     return_tensors="""pt"""   )
                     self.assertEqual(batch.input_ids.shape[1]   ,     3   )
                     self.assertEqual(batch.labels.shape[1]   ,     3   )

                     lowerCAmelCase__      :			int   =	tokenizer.prepare_seqaseq_batch(
                         src_texts=UpperCamelCase   ,     max_length=3   ,     max_target_length=10   ,     return_tensors="""pt"""   )
                     self.assertEqual(batch_encoder_only.input_ids.shape[1]   ,     3   )
                     self.assertEqual(batch_encoder_only.attention_mask.shape[1]   ,     3   )
                     self.assertNotIn("""decoder_input_ids"""   ,     UpperCamelCase   )






   @unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""   )
   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						Dict:

         """simple docstring"""
         pass






   def 	_lowerCAmelCase   (		self       :		str   )			->						Union[str, Any]:

         """simple docstring"""
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
                     lowerCAmelCase__      :			str   =	[AddedToken("""<special>"""   ,     lstrip=UpperCamelCase   )]

                     lowerCAmelCase__      :			Optional[Any]   =	self.rust_tokenizer_class.from_pretrained(
                         UpperCamelCase   ,     additional_special_tokens=UpperCamelCase   ,     **UpperCamelCase   )
                     lowerCAmelCase__      :			Dict   =	tokenizer_r.encode("""Hey this is a <special> token"""   )

                     lowerCAmelCase__      :			Dict   =	tokenizer_r.encode("""<special>"""   ,     add_special_tokens=UpperCamelCase   )[0]

                     self.assertTrue(special_token_id in r_output   )

                     if self.test_slow_tokenizer:
                           lowerCAmelCase__      :			List[Any]   =	self.rust_tokenizer_class.from_pretrained(
                               UpperCamelCase   ,     additional_special_tokens=UpperCamelCase   ,     **UpperCamelCase   ,     )
                           lowerCAmelCase__      :			Dict   =	self.tokenizer_class.from_pretrained(
                               UpperCamelCase   ,     additional_special_tokens=UpperCamelCase   ,     **UpperCamelCase   )

                           lowerCAmelCase__      :			Optional[int]   =	tokenizer_p.encode("""Hey this is a <special> token"""   )

                           lowerCAmelCase__      :			Dict   =	tokenizer_cr.encode("""Hey this is a <special> token"""   )

                           self.assertEqual(UpperCamelCase   ,     UpperCamelCase   )
                           self.assertEqual(UpperCamelCase   ,     UpperCamelCase   )
                           self.assertTrue(special_token_id in p_output   )
                           self.assertTrue(special_token_id in cr_output   )






@require_torch
@require_sentencepiece
@require_tokenizers
class _lowerCamelCase    (					unittest.TestCase						):
   _lowerCamelCase    :int     				=     "facebook/nllb-200-distilled-600M"
   _lowerCamelCase    :List[str]     				=     [
       " UN Chief Says There Is No Military Solution in Syria",
       " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
   ]
   _lowerCamelCase    :Optional[Any]     				=     [
       "Şeful ONU declară că nu există o soluţie militară în Siria",
       "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
       " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
       " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
   ]
   _lowerCamelCase    :Tuple     				=     [
       256047,
       16297,
       134408,
       8165,
       248066,
       14734,
       950,
       1135,
       105721,
       3573,
       83,
       27352,
       108,
       49486,
       2,
   ]






   @classmethod
   def 	_lowerCAmelCase   (		cls       :		Optional[Any]   )			->						Any:

         """simple docstring"""
         lowerCAmelCase__      :			NllbTokenizer   =	NllbTokenizer.from_pretrained(
             cls.checkpoint_name   ,     src_lang="""eng_Latn"""   ,     tgt_lang="""ron_Latn"""   )
         lowerCAmelCase__      :			Optional[Any]   =	1
         return cls






   def 	_lowerCAmelCase   (		self       :		Any   )			->						int:

         """simple docstring"""
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]   ,     25_60_01   )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]   ,     25_60_02   )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]   ,     25_60_57   )






   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	self.tokenizer.batch_encode_plus(self.src_text   ).input_ids[0]
         self.assertListEqual(self.expected_src_tokens   ,     UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						Optional[int]:

         """simple docstring"""
         self.assertIn(UpperCamelCase   ,     self.tokenizer.all_special_ids   )
         # fmt: off
         lowerCAmelCase__      :			str   =	[RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
         # fmt: on

         lowerCAmelCase__      :			Any   =	self.tokenizer.decode(UpperCamelCase   ,     skip_special_tokens=UpperCamelCase   )
         lowerCAmelCase__      :			Tuple   =	self.tokenizer.decode(generated_ids[1:]   ,     skip_special_tokens=UpperCamelCase   )
         self.assertEqual(UpperCamelCase   ,     UpperCamelCase   )
         self.assertNotIn(self.tokenizer.eos_token   ,     UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		Optional[int]   )			->						List[str]:

         """simple docstring"""
         lowerCAmelCase__      :			List[Any]   =	["""this is gunna be a long sentence """ * 20]
         assert isinstance(src_text[0]   ,     UpperCamelCase   )
         lowerCAmelCase__      :			int   =	10
         lowerCAmelCase__      :			Any   =	self.tokenizer(UpperCamelCase   ,     max_length=UpperCamelCase   ,     truncation=UpperCamelCase   ).input_ids[0]
         self.assertEqual(ids[-1]   ,     2   )
         self.assertEqual(ids[0]   ,     UpperCamelCase   )
         self.assertEqual(len(UpperCamelCase   )   ,     UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		Any   )			->						Any:

         """simple docstring"""
         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]   )   ,     [25_62_03, 3]   )






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			List[Any]   =	tempfile.mkdtemp()
         lowerCAmelCase__      :			int   =	self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(UpperCamelCase   )
         lowerCAmelCase__      :			Union[str, Any]   =	NllbTokenizer.from_pretrained(UpperCamelCase   )
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,     UpperCamelCase   )






   @require_torch
   def 	_lowerCAmelCase   (		self       :		int   )			->						List[str]:

         """simple docstring"""
         lowerCAmelCase__      :			Optional[int]   =	self.tokenizer(
             self.src_text   ,     text_target=self.tgt_text   ,     padding=UpperCamelCase   ,     truncation=UpperCamelCase   ,     max_length=len(self.expected_src_tokens   )   ,     return_tensors="""pt"""   ,     )
         lowerCAmelCase__      :			int   =	shift_tokens_right(
             batch["""labels"""]   ,     self.tokenizer.pad_token_id   ,     self.tokenizer.lang_code_to_id["""ron_Latn"""]   )

         self.assertIsInstance(UpperCamelCase   ,     UpperCamelCase   )

         self.assertEqual((2, 15)   ,     batch.input_ids.shape   )
         self.assertEqual((2, 15)   ,     batch.attention_mask.shape   )
         lowerCAmelCase__      :			Dict   =	batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens   ,     UpperCamelCase   )
         self.assertEqual(UpperCamelCase   ,     batch.decoder_input_ids[0, 0]   )  # EOS
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens   ,     [EN_CODE]   )
         self.assertEqual(self.tokenizer.suffix_tokens   ,     [self.tokenizer.eos_token_id]   )






   def 	_lowerCAmelCase   (		self       :		Optional[int]   )			->						int:

         """simple docstring"""
         lowerCAmelCase__      :			str   =	self.tokenizer(self.src_text   ,     padding=UpperCamelCase   ,     truncation=UpperCamelCase   ,     max_length=3   ,     return_tensors="""pt"""   )
         lowerCAmelCase__      :			Any   =	self.tokenizer(
             text_target=self.tgt_text   ,     padding=UpperCamelCase   ,     truncation=UpperCamelCase   ,     max_length=10   ,     return_tensors="""pt"""   )
         lowerCAmelCase__      :			str   =	targets["""input_ids"""]
         lowerCAmelCase__      :			Any   =	shift_tokens_right(
             UpperCamelCase   ,     self.tokenizer.pad_token_id   ,     decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]   ,     )

         self.assertEqual(batch.input_ids.shape[1]   ,     3   )
         self.assertEqual(batch.decoder_input_ids.shape[1]   ,     10   )






   @require_torch
   def 	_lowerCAmelCase   (		self       :		Dict   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Any   =	self.tokenizer._build_translation_inputs(
             """A test"""   ,     return_tensors="""pt"""   ,     src_lang="""eng_Latn"""   ,     tgt_lang="""fra_Latn"""   )

         self.assertEqual(
             nested_simplify(UpperCamelCase   )   ,     {
                 # A, test, EOS, en_XX
                 """input_ids""": [[25_60_47, 70, 73_56, 2]],
                 """attention_mask""": [[1, 1, 1, 1]],
                 # ar_AR
                 """forced_bos_token_id""": 25_60_57,
             }   ,     )






   @require_torch
   def 	_lowerCAmelCase   (		self       :		Dict   )			->						Tuple:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	True
         lowerCAmelCase__      :			str   =	self.tokenizer(
             """UN Chief says there is no military solution in Syria"""   ,     src_lang="""eng_Latn"""   ,     tgt_lang="""fra_Latn"""   )
         self.assertEqual(
             inputs.input_ids   ,     [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]   )

         lowerCAmelCase__      :			List[str]   =	False
         lowerCAmelCase__      :			Union[str, Any]   =	self.tokenizer(
             """UN Chief says there is no military solution in Syria"""   ,     src_lang="""eng_Latn"""   ,     tgt_lang="""fra_Latn"""   )
         self.assertEqual(
             inputs.input_ids   ,     [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]   )





