





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_mbart import MBartTokenizer
else:
      SCREAMING_SNAKE_CASE_						       =	None


SCREAMING_SNAKE_CASE_						       =	logging.get_logger(__name__)


SCREAMING_SNAKE_CASE_						       =	{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE_						       =	{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-en-ro""": """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json""",
        """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json""",
    },
}

SCREAMING_SNAKE_CASE_						       =	{
    """facebook/mbart-large-en-ro""": 1_0_2_4,
    """facebook/mbart-large-cc25""": 1_0_2_4,
}

# fmt: off
SCREAMING_SNAKE_CASE_						       =	["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]

class 		UpperCamelCase__			(				lowerCAmelCase_       ):



     '''simple docstring'''

     __snake_case   :     Tuple		     =      VOCAB_FILES_NAMES
     __snake_case   :     Tuple		     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __snake_case   :     str		     =      PRETRAINED_VOCAB_FILES_MAP
     __snake_case   :     Union[str, Any]		     =      ["input_ids", "attention_mask"]
     __snake_case   :     Optional[Any]		     =      MBartTokenizer

     __snake_case   :     List[int]		     =      []
     __snake_case   :     List[int]		     =      []






     def __init__(							self						:   Optional[int]						,lowerCamelCase__						:   Any=None						,lowerCamelCase__						:   str=None						,lowerCamelCase__						:   Dict="<s>"						,lowerCamelCase__						:   Any="</s>"						,lowerCamelCase__						:   Optional[int]="</s>"						,lowerCamelCase__						:   int="<s>"						,lowerCamelCase__						:   int="<unk>"						,lowerCamelCase__						:   List[str]="<pad>"						,lowerCamelCase__						:   List[str]="<mask>"						,lowerCamelCase__						:   str=None						,lowerCamelCase__						:   int=None						,lowerCamelCase__						:   Optional[int]=None						,**lowerCamelCase__						:   Dict						,)					->		Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						AddedToken(lowerCamelCase__						,lstrip=lowerCamelCase__						,rstrip=lowerCamelCase__	) if isinstance(lowerCamelCase__						,lowerCamelCase__	) else mask_token

           super().__init__(
               vocab_file=lowerCamelCase__						,tokenizer_file=lowerCamelCase__						,bos_token=lowerCamelCase__						,eos_token=lowerCamelCase__						,sep_token=lowerCamelCase__						,cls_token=lowerCamelCase__						,unk_token=lowerCamelCase__						,pad_token=lowerCamelCase__						,mask_token=lowerCamelCase__						,src_lang=lowerCamelCase__						,tgt_lang=lowerCamelCase__						,additional_special_tokens=lowerCamelCase__						,**lowerCamelCase__						,)

           SCREAMING_SNAKE_CASE		=						vocab_file
           SCREAMING_SNAKE_CASE		=						False if not self.vocab_file else True

           SCREAMING_SNAKE_CASE		=						FAIRSEQ_LANGUAGE_CODES.copy()

           if additional_special_tokens is not None:
                 # Only add those special tokens if they are not already there.
                 _additional_special_tokens.extend(
                     [t for t in additional_special_tokens if t not in _additional_special_tokens]	)

           self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}	)
           SCREAMING_SNAKE_CASE		=						{
               lang_code: self.convert_tokens_to_ids(lowerCamelCase__	) for lang_code in FAIRSEQ_LANGUAGE_CODES
           }

           SCREAMING_SNAKE_CASE		=						src_lang if src_lang is not None else """en_XX"""
           SCREAMING_SNAKE_CASE		=						self.convert_tokens_to_ids(self._src_lang	)
           SCREAMING_SNAKE_CASE		=						tgt_lang
           self.set_src_lang_special_tokens(self._src_lang	)






     @property
     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		str:




           '''simple docstring'''


           return self._src_lang






     @src_lang.setter
     def     SCREAMING_SNAKE_CASE__  (							self						:   Tuple						,lowerCamelCase__						:   str	)					->		None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						new_src_lang
           self.set_src_lang_special_tokens(self._src_lang	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict						,lowerCamelCase__						:   List[int]						,lowerCamelCase__						:   Optional[List[int]] = None	)					->		List[int]:




           '''simple docstring'''


           if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[str]						,lowerCamelCase__						:   List[int]						,lowerCamelCase__						:   Optional[List[int]] = None	)					->		List[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						[self.sep_token_id]
           SCREAMING_SNAKE_CASE		=						[self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep	) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]






     def     SCREAMING_SNAKE_CASE__  (							self						:   int						,lowerCamelCase__						:   Tuple						,lowerCamelCase__						:   str						,lowerCamelCase__						:   Optional[str]						,lowerCamelCase__						:   Optional[str]						,**lowerCamelCase__						:   Any	)					->		List[str]:




           '''simple docstring'''


           if src_lang is None or tgt_lang is None:
                 raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""	)
           SCREAMING_SNAKE_CASE		=						src_lang
           SCREAMING_SNAKE_CASE		=						self(lowerCamelCase__						,add_special_tokens=lowerCamelCase__						,return_tensors=lowerCamelCase__						,**lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						self.convert_tokens_to_ids(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						tgt_lang_id
           return inputs






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]						,lowerCamelCase__						:   List[str]						,lowerCamelCase__						:   str = "en_XX"						,lowerCamelCase__						:   Optional[List[str]] = None						,lowerCamelCase__						:   str = "ro_RO"						,**lowerCamelCase__						:   Any						,)					->		BatchEncoding:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						src_lang
           SCREAMING_SNAKE_CASE		=						tgt_lang
           return super().prepare_seqaseq_batch(lowerCamelCase__						,lowerCamelCase__						,**lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[Any]	)					->		List[str]:




           '''simple docstring'''


           return self.set_src_lang_special_tokens(self.src_lang	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[Any]	)					->		List[Any]:




           '''simple docstring'''


           return self.set_tgt_lang_special_tokens(self.tgt_lang	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]						,lowerCamelCase__						:   List[str]	)					->		None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.convert_tokens_to_ids(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						[]
           SCREAMING_SNAKE_CASE		=						[self.eos_token_id, self.cur_lang_code]

           SCREAMING_SNAKE_CASE		=						self.convert_ids_to_tokens(self.prefix_tokens	)
           SCREAMING_SNAKE_CASE		=						self.convert_ids_to_tokens(self.suffix_tokens	)

           SCREAMING_SNAKE_CASE		=						processors.TemplateProcessing(
               single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str						,pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str						,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,self.prefix_tokens + self.suffix_tokens	)	)						,)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Tuple						,lowerCamelCase__						:   str	)					->		None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.convert_tokens_to_ids(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						[]
           SCREAMING_SNAKE_CASE		=						[self.eos_token_id, self.cur_lang_code]

           SCREAMING_SNAKE_CASE		=						self.convert_ids_to_tokens(self.prefix_tokens	)
           SCREAMING_SNAKE_CASE		=						self.convert_ids_to_tokens(self.suffix_tokens	)

           SCREAMING_SNAKE_CASE		=						processors.TemplateProcessing(
               single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str						,pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str						,special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,self.prefix_tokens + self.suffix_tokens	)	)						,)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]						,lowerCamelCase__						:   str						,lowerCamelCase__						:   Optional[str] = None	)					->		Tuple[str]:




           '''simple docstring'''


           if not self.can_save_slow_tokenizer:
                 raise ValueError(
                     """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                     """tokenizer."""	)

           if not os.path.isdir(lowerCamelCase__	):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""	)
                 return
           SCREAMING_SNAKE_CASE		=						os.path.join(
               lowerCamelCase__						,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

           if os.path.abspath(self.vocab_file	) != os.path.abspath(lowerCamelCase__	):
                 copyfile(self.vocab_file						,lowerCamelCase__	)

           return (out_vocab_file,)









import inspect
import unittest

from transformers import RegNetConfig, is_flax_available
from transformers.testing_utils import require_flax, slow
from transformers.utils import cached_property, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor


if is_flax_available():
      import jax
      import jax.numpy as jnp

      from transformers.models.regnet.modeling_flax_regnet import FlaxRegNetForImageClassification, FlaxRegNetModel

if is_vision_available():
      from PIL import Image

      from transformers import AutoImageProcessor

class 		UpperCamelCase__			(				unittest.TestCase       ):



     '''simple docstring'''

     def __init__(							self						:   Optional[int]						,lowerCamelCase__						:   List[str]						,lowerCamelCase__						:   Optional[Any]=3						,lowerCamelCase__						:   List[str]=32						,lowerCamelCase__						:   List[Any]=3						,lowerCamelCase__						:   str=10						,lowerCamelCase__						:   Any=[10, 20, 30, 40]						,lowerCamelCase__						:   Optional[Any]=[1, 1, 2, 1]						,lowerCamelCase__						:   Union[str, Any]=True						,lowerCamelCase__						:   int=True						,lowerCamelCase__						:   Tuple="relu"						,lowerCamelCase__						:   Dict=3						,lowerCamelCase__						:   Optional[int]=None						,)					->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						parent
           SCREAMING_SNAKE_CASE		=						batch_size
           SCREAMING_SNAKE_CASE		=						image_size
           SCREAMING_SNAKE_CASE		=						num_channels
           SCREAMING_SNAKE_CASE		=						embeddings_size
           SCREAMING_SNAKE_CASE		=						hidden_sizes
           SCREAMING_SNAKE_CASE		=						depths
           SCREAMING_SNAKE_CASE		=						is_training
           SCREAMING_SNAKE_CASE		=						use_labels
           SCREAMING_SNAKE_CASE		=						hidden_act
           SCREAMING_SNAKE_CASE		=						num_labels
           SCREAMING_SNAKE_CASE		=						scope
           SCREAMING_SNAKE_CASE		=						len(lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Tuple	)					->		Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]	)

           SCREAMING_SNAKE_CASE		=						self.get_config()

           return config, pixel_values






     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		Optional[int]:




           '''simple docstring'''


           return RegNetConfig(
               num_channels=self.num_channels						,embeddings_size=self.embeddings_size						,hidden_sizes=self.hidden_sizes						,depths=self.depths						,hidden_act=self.hidden_act						,num_labels=self.num_labels						,image_size=self.image_size						,)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]						,lowerCamelCase__						:   int						,lowerCamelCase__						:   Union[str, Any]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						FlaxRegNetModel(config=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						model(lowerCamelCase__	)

           # Output shape (b, c, h, w)
           self.parent.assertEqual(
               result.last_hidden_state.shape						,(self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)						,)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]						,lowerCamelCase__						:   Optional[int]						,lowerCamelCase__						:   Optional[Any]	)					->		Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.num_labels
           SCREAMING_SNAKE_CASE		=						FlaxRegNetForImageClassification(config=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						model(lowerCamelCase__	)
           self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.num_labels)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.prepare_config_and_inputs()
           SCREAMING_SNAKE_CASE, SCREAMING_SNAKE_CASE		=						config_and_inputs
           SCREAMING_SNAKE_CASE		=						{"""pixel_values""": pixel_values}
           return config, inputs_dict

@require_flax
class 		UpperCamelCase__			(				lowerCAmelCase_							,				unittest.TestCase       ):



     '''simple docstring'''

     __snake_case   :     Union[str, Any]		     =      (FlaxRegNetModel, FlaxRegNetForImageClassification) if is_flax_available() else ()

     __snake_case   :     Tuple		     =      False
     __snake_case   :     int		     =      False
     __snake_case   :     Tuple		     =      False






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]	)					->		None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						FlaxRegNetModelTester(self	)
           SCREAMING_SNAKE_CASE		=						ConfigTester(self						,config_class=lowerCamelCase__						,has_text_modality=lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Optional[int]:




           '''simple docstring'''


           self.create_and_test_config_common_properties()
           self.config_tester.create_and_test_config_to_json_string()
           self.config_tester.create_and_test_config_to_json_file()
           self.config_tester.create_and_test_config_from_and_save_pretrained()
           self.config_tester.create_and_test_config_with_num_labels()
           self.config_tester.check_config_can_be_init_without_params()
           self.config_tester.check_config_arguments_init()






     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		List[str]:




           '''simple docstring'''


           return






     def     SCREAMING_SNAKE_CASE__  (							self						:   Tuple	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_image_classification(*lowerCamelCase__	)






     @unittest.skip(reason="""RegNet does not use inputs_embeds"""	)
     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		Any:




           '''simple docstring'''


           pass






     @unittest.skip(reason="""RegNet does not support input and output embeddings"""	)
     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		Any:




           '''simple docstring'''


           pass






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE, SCREAMING_SNAKE_CASE		=						self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                 SCREAMING_SNAKE_CASE		=						model_class(lowerCamelCase__	)
                 SCREAMING_SNAKE_CASE		=						inspect.signature(model.__call__	)
                 # signature.parameters is an OrderedDict => so arg_names order is deterministic
                 SCREAMING_SNAKE_CASE		=						[*signature.parameters.keys()]

                 SCREAMING_SNAKE_CASE		=						["""pixel_values"""]
                 self.assertListEqual(arg_names[:1]						,lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		int:




           '''simple docstring'''


           def check_hidden_states_output(lowerCamelCase__						:   Tuple						,lowerCamelCase__						:   Union[str, Any]						,lowerCamelCase__						:   Optional[int]	):
                 SCREAMING_SNAKE_CASE		=						model_class(lowerCamelCase__	)

                 SCREAMING_SNAKE_CASE		=						model(**self._prepare_for_class(lowerCamelCase__						,lowerCamelCase__	)	)

                 SCREAMING_SNAKE_CASE		=						outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

                 SCREAMING_SNAKE_CASE		=						self.model_tester.num_stages
                 self.assertEqual(len(lowerCamelCase__	)						,expected_num_stages + 1	)

           SCREAMING_SNAKE_CASE, SCREAMING_SNAKE_CASE		=						self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                 SCREAMING_SNAKE_CASE		=						True
                 check_hidden_states_output(lowerCamelCase__						,lowerCamelCase__						,lowerCamelCase__	)

                 # check that output_hidden_states also work using config
                 del inputs_dict["output_hidden_states"]
                 SCREAMING_SNAKE_CASE		=						True

                 check_hidden_states_output(lowerCamelCase__						,lowerCamelCase__						,lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE, SCREAMING_SNAKE_CASE		=						self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                 with self.subTest(model_class.__name__	):
                       SCREAMING_SNAKE_CASE		=						self._prepare_for_class(lowerCamelCase__						,lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						model_class(lowerCamelCase__	)

                       @jax.jit
                       def model_jitted(lowerCamelCase__						:   Dict						,**lowerCamelCase__						:   Optional[Any]	):
                             return model(pixel_values=lowerCamelCase__						,**lowerCamelCase__	)

                       with self.subTest("""JIT Enabled"""	):
                             SCREAMING_SNAKE_CASE		=						model_jitted(**lowerCamelCase__	).to_tuple()

                       with self.subTest("""JIT Disabled"""	):
                             with jax.disable_jit():
                                   SCREAMING_SNAKE_CASE		=						model_jitted(**lowerCamelCase__	).to_tuple()

                       self.assertEqual(len(lowerCamelCase__	)						,len(lowerCamelCase__	)	)
                       for jitted_output, output in zip(lowerCamelCase__						,lowerCamelCase__	):
                             self.assertEqual(jitted_output.shape						,output.shape	)



def        __lowercase       (				)			->       Union[str, Any]:


      '''simple docstring'''


      SCREAMING_SNAKE_CASE		=						Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""     )
      return image



@require_flax
class 		UpperCamelCase__			(				unittest.TestCase       ):



     '''simple docstring'''

     @cached_property
     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		str:




           '''simple docstring'''


           return AutoImageProcessor.from_pretrained("""facebook/regnet-y-040"""	) if is_vision_available() else None






     @slow
     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		List[str]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						FlaxRegNetForImageClassification.from_pretrained("""facebook/regnet-y-040"""	)

           SCREAMING_SNAKE_CASE		=						self.default_image_processor
           SCREAMING_SNAKE_CASE		=						prepare_img()
           SCREAMING_SNAKE_CASE		=						image_processor(images=lowerCamelCase__						,return_tensors="""np"""	)

           SCREAMING_SNAKE_CASE		=						model(**lowerCamelCase__	)

           # verify the logits
           SCREAMING_SNAKE_CASE		=						(1, 1000)
           self.assertEqual(outputs.logits.shape						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						jnp.array([-0.4180, -1.5051, -3.4836]	)

           self.assertTrue(jnp.allclose(outputs.logits[0, :3]						,lowerCamelCase__						,atol=1e-4	)	)


