






'''simple docstring'''
import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast






class        __SCREAMING_SNAKE_CASE (     unittest.TestCase			):




 def        __magic_name__		( self		:							Tuple )		->  List[str]:
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=0




 @slow
 def        __magic_name__		( self		:							Union[str, Any] )		->  Union[str, Any]:
       for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
             SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=AutoTokenizer.from_pretrained(__lowercase )
             self.assertIsNotNone(__lowercase )
             self.assertIsInstance(__lowercase   , (BertTokenizer, BertTokenizerFast) )
             self.assertGreater(len(__lowercase )   , 0 )

       for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
             SCREAMING_SNAKE_CASE__       :			int													=AutoTokenizer.from_pretrained(__lowercase )
             self.assertIsNotNone(__lowercase )
             self.assertIsInstance(__lowercase   , (GPTaTokenizer, GPTaTokenizerFast) )
             self.assertGreater(len(__lowercase )   , 0 )




 def        __magic_name__		( self		:							Tuple )		->  Tuple:
       SCREAMING_SNAKE_CASE__       :			Tuple													=AutoTokenizer.from_pretrained(__lowercase )
       self.assertIsInstance(__lowercase   , (BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size   , 12 )




 def        __magic_name__		( self		:							Optional[Any] )		->  Optional[int]:
       SCREAMING_SNAKE_CASE__       :			List[Any]													=AutoTokenizer.from_pretrained(__lowercase )
       self.assertIsInstance(__lowercase   , (RobertaTokenizer, RobertaTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size   , 20 )




 def        __magic_name__		( self		:							Any )		->  Any:
       SCREAMING_SNAKE_CASE__       :			List[str]													=AutoConfig.from_pretrained(__lowercase )
       self.assertIsInstance(__lowercase   , __lowercase )
       # Check that tokenizer_type ≠ model_type
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=AutoTokenizer.from_pretrained(__lowercase   , config=__lowercase )
       self.assertIsInstance(__lowercase   , (BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size   , 12 )




 def        __magic_name__		( self		:							Optional[int] )		->  List[str]:
       with tempfile.TemporaryDirectory() as tmp_dir:
             shutil.copy('''./tests/fixtures/vocab.txt'''   , os.path.join(__lowercase   , '''vocab.txt''' ) )

             SCREAMING_SNAKE_CASE__       :			List[Any]													=AutoTokenizer.from_pretrained(__lowercase   , tokenizer_type='''bert'''   , use_fast=__lowercase )
             self.assertIsInstance(__lowercase   , __lowercase )

       with tempfile.TemporaryDirectory() as tmp_dir:
             shutil.copy('''./tests/fixtures/vocab.json'''   , os.path.join(__lowercase   , '''vocab.json''' ) )
             shutil.copy('''./tests/fixtures/merges.txt'''   , os.path.join(__lowercase   , '''merges.txt''' ) )

             SCREAMING_SNAKE_CASE__       :			Optional[Any]													=AutoTokenizer.from_pretrained(__lowercase   , tokenizer_type='''gpt2'''   , use_fast=__lowercase )
             self.assertIsInstance(__lowercase   , __lowercase )




 @require_tokenizers
 def        __magic_name__		( self		:							Tuple )		->  Optional[int]:
       with tempfile.TemporaryDirectory() as tmp_dir:
             shutil.copy('''./tests/fixtures/vocab.txt'''   , os.path.join(__lowercase   , '''vocab.txt''' ) )

             SCREAMING_SNAKE_CASE__       :			List[Any]													=AutoTokenizer.from_pretrained(__lowercase   , tokenizer_type='''bert''' )
             self.assertIsInstance(__lowercase   , __lowercase )

       with tempfile.TemporaryDirectory() as tmp_dir:
             shutil.copy('''./tests/fixtures/vocab.json'''   , os.path.join(__lowercase   , '''vocab.json''' ) )
             shutil.copy('''./tests/fixtures/merges.txt'''   , os.path.join(__lowercase   , '''merges.txt''' ) )

             SCREAMING_SNAKE_CASE__       :			Any													=AutoTokenizer.from_pretrained(__lowercase   , tokenizer_type='''gpt2''' )
             self.assertIsInstance(__lowercase   , __lowercase )




 def        __magic_name__		( self		:							Optional[int] )		->  int:
       with pytest.raises(__lowercase ):
             AutoTokenizer.from_pretrained('''./'''   , tokenizer_type='''xxx''' )




 @require_tokenizers
 def        __magic_name__		( self		:							Union[str, Any] )		->  Dict:
       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
             SCREAMING_SNAKE_CASE__       :			List[Any]													=tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased''' )
             self.assertIsInstance(__lowercase   , (BertTokenizer, BertTokenizerFast) )

             if isinstance(__lowercase   , __lowercase ):
                   self.assertEqual(tokenizer.basic_tokenizer.do_lower_case   , __lowercase )
             else:
                   self.assertEqual(tokenizer.do_lower_case   , __lowercase )

             self.assertEqual(tokenizer.model_max_length   , 5_12 )




 @require_tokenizers
 def        __magic_name__		( self		:							str )		->  List[str]:
       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
             with self.assertRaisesRegex(
                 __lowercase   , '''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''   , ):
                   SCREAMING_SNAKE_CASE__       :			Optional[int]													=tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists''' )




 def        __magic_name__		( self		:							Union[str, Any] )		->  List[Any]:
       # tests: https://github.com/huggingface/transformers/pull/13251
       # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
       # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
       SCREAMING_SNAKE_CASE__       :			Tuple													=TOKENIZER_MAPPING.values()
       SCREAMING_SNAKE_CASE__       :			int													=[]

       for slow_tok, fast_tok in tokenizers:
             if slow_tok is not None:
                   tokenizer_names.append(slow_tok.__name__ )

             if fast_tok is not None:
                   tokenizer_names.append(fast_tok.__name__ )

       for tokenizer_name in tokenizer_names:
             # must find the right class
             tokenizer_class_from_name(__lowercase )




 @require_tokenizers
 def        __magic_name__		( self		:							List[str] )		->  Union[str, Any]:
       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''   , use_fast=__lowercase )   , __lowercase )
       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' )   , __lowercase )




 @require_tokenizers
 def        __magic_name__		( self		:							List[str] )		->  List[str]:
       SCREAMING_SNAKE_CASE__       :			List[Any]													=AutoTokenizer.from_pretrained('''distilbert-base-uncased'''   , do_lower_case=__lowercase )
       SCREAMING_SNAKE_CASE__       :			str													='''Hello, world. How are you?'''
       SCREAMING_SNAKE_CASE__       :			str													=tokenizer.tokenize(__lowercase )
       self.assertEqual('''[UNK]'''   , tokens[0] )

       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''   , do_lower_case=__lowercase )
       SCREAMING_SNAKE_CASE__       :			Optional[int]													=tokenizer.tokenize(__lowercase )
       self.assertEqual('''[UNK]'''   , tokens[0] )




 @require_tokenizers
 def        __magic_name__		( self		:							Dict )		->  Any:
       SCREAMING_SNAKE_CASE__       :			List[str]													=AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config''' )
       self.assertEqual(type(__lowercase )   , __lowercase )
       self.assertEqual(tokenizer.model_max_length   , 5_12 )
       self.assertEqual(tokenizer.vocab_size   , 3_00_00 )
       self.assertEqual(tokenizer.unk_token   , '''[UNK]''' )
       self.assertEqual(tokenizer.padding_side   , '''right''' )
       self.assertEqual(tokenizer.truncation_side   , '''right''' )




 def        __magic_name__		( self		:							Optional[Any] )		->  Optional[int]:
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=AutoTokenizer.from_pretrained(__lowercase )
       self.assertIsInstance(__lowercase   , (BertTokenizer, BertTokenizerFast) )
       with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(__lowercase )
             SCREAMING_SNAKE_CASE__       :			Optional[Any]													=AutoTokenizer.from_pretrained(__lowercase )

       self.assertIsInstance(__lowercase   , tokenizer.__class__ )
       self.assertEqual(tokenizera.vocab_size   , 12 )




 def        __magic_name__		( self		:							List[Any] )		->  List[str]:
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=AutoTokenizer.from_pretrained('''ctrl''' )
       # There is no fast CTRL so this always gives us a slow tokenizer.
       self.assertIsInstance(__lowercase   , __lowercase )




 def        __magic_name__		( self		:							str )		->  List[str]:
       # Check we can load the tokenizer config of an online model.
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=get_tokenizer_config('''bert-base-cased''' )
       SCREAMING_SNAKE_CASE__       :			List[Any]													=config.pop('''_commit_hash'''   , __lowercase )
       # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
       self.assertEqual(__lowercase   , {'''do_lower_case''': False} )

       # This model does not have a tokenizer_config so we get back an empty dict.
       SCREAMING_SNAKE_CASE__       :			str													=get_tokenizer_config(__lowercase )
       self.assertDictEqual(__lowercase   , {} )

       # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
       SCREAMING_SNAKE_CASE__       :			int													=AutoTokenizer.from_pretrained(__lowercase )
       with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(__lowercase )
             SCREAMING_SNAKE_CASE__       :			Any													=get_tokenizer_config(__lowercase )

       # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
       self.assertEqual(config['''tokenizer_class''']   , '''BertTokenizer''' )




 def        __magic_name__		( self		:							Any )		->  Tuple:
       try:
             AutoConfig.register('''custom'''   , __lowercase )

             AutoTokenizer.register(__lowercase   , slow_tokenizer_class=__lowercase )
             # Trying to register something existing in the Transformers library will raise an error
             with self.assertRaises(__lowercase ):
                   AutoTokenizer.register(__lowercase   , slow_tokenizer_class=__lowercase )

             SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=CustomTokenizer.from_pretrained(__lowercase )
             with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(__lowercase )

                   SCREAMING_SNAKE_CASE__       :			Optional[int]													=AutoTokenizer.from_pretrained(__lowercase )
                   self.assertIsInstance(__lowercase   , __lowercase )

       finally:
             if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
             if CustomConfig in TOKENIZER_MAPPING._extra_content:
                   del TOKENIZER_MAPPING._extra_content[CustomConfig]




 @require_tokenizers
 def        __magic_name__		( self		:							int )		->  Any:
       try:
             AutoConfig.register('''custom'''   , __lowercase )

             # Can register in two steps
             AutoTokenizer.register(__lowercase   , slow_tokenizer_class=__lowercase )
             self.assertEqual(TOKENIZER_MAPPING[CustomConfig]   , (CustomTokenizer, None) )
             AutoTokenizer.register(__lowercase   , fast_tokenizer_class=__lowercase )
             self.assertEqual(TOKENIZER_MAPPING[CustomConfig]   , (CustomTokenizer, CustomTokenizerFast) )

             del TOKENIZER_MAPPING._extra_content[CustomConfig]
             # Can register in one step
             AutoTokenizer.register(
                 __lowercase   , slow_tokenizer_class=__lowercase   , fast_tokenizer_class=__lowercase )
             self.assertEqual(TOKENIZER_MAPPING[CustomConfig]   , (CustomTokenizer, CustomTokenizerFast) )

             # Trying to register something existing in the Transformers library will raise an error
             with self.assertRaises(__lowercase ):
                   AutoTokenizer.register(__lowercase   , fast_tokenizer_class=__lowercase )

             # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
             # and that model does not have a tokenizer.json
             with tempfile.TemporaryDirectory() as tmp_dir:
                   SCREAMING_SNAKE_CASE__       :			str													=BertTokenizerFast.from_pretrained(__lowercase )
                   bert_tokenizer.save_pretrained(__lowercase )
                   SCREAMING_SNAKE_CASE__       :			Any													=CustomTokenizerFast.from_pretrained(__lowercase )

             with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(__lowercase )

                   SCREAMING_SNAKE_CASE__       :			Optional[Any]													=AutoTokenizer.from_pretrained(__lowercase )
                   self.assertIsInstance(__lowercase   , __lowercase )

                   SCREAMING_SNAKE_CASE__       :			List[Any]													=AutoTokenizer.from_pretrained(__lowercase   , use_fast=__lowercase )
                   self.assertIsInstance(__lowercase   , __lowercase )

       finally:
             if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
             if CustomConfig in TOKENIZER_MAPPING._extra_content:
                   del TOKENIZER_MAPPING._extra_content[CustomConfig]




 def        __magic_name__		( self		:							Union[str, Any] )		->  Tuple:
       # If remote code is not set, we will time out when asking whether to load the model.
       with self.assertRaises(__lowercase ):
             SCREAMING_SNAKE_CASE__       :			int													=AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
       # If remote code is disabled, we can't load this config.
       with self.assertRaises(__lowercase ):
             SCREAMING_SNAKE_CASE__       :			Optional[int]													=AutoTokenizer.from_pretrained(
                 '''hf-internal-testing/test_dynamic_tokenizer'''   , trust_remote_code=__lowercase )

       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''   , trust_remote_code=__lowercase )
       self.assertTrue(tokenizer.special_attribute_present )
       # Test tokenizer can be reloaded.
       with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(__lowercase )
             SCREAMING_SNAKE_CASE__       :			Dict													=AutoTokenizer.from_pretrained(__lowercase   , trust_remote_code=__lowercase )
       self.assertTrue(reloaded_tokenizer.special_attribute_present )

       if is_tokenizers_available():
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizerFast''' )
             self.assertEqual(reloaded_tokenizer.__class__.__name__   , '''NewTokenizerFast''' )

             # Test we can also load the slow version
             SCREAMING_SNAKE_CASE__       :			str													=AutoTokenizer.from_pretrained(
                 '''hf-internal-testing/test_dynamic_tokenizer'''   , trust_remote_code=__lowercase   , use_fast=__lowercase )
             self.assertTrue(tokenizer.special_attribute_present )
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizer''' )
             # Test tokenizer can be reloaded.
             with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(__lowercase )
                   SCREAMING_SNAKE_CASE__       :			Optional[Any]													=AutoTokenizer.from_pretrained(__lowercase   , trust_remote_code=__lowercase   , use_fast=__lowercase )
             self.assertEqual(reloaded_tokenizer.__class__.__name__   , '''NewTokenizer''' )
             self.assertTrue(reloaded_tokenizer.special_attribute_present )
       else:
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizer''' )
             self.assertEqual(reloaded_tokenizer.__class__.__name__   , '''NewTokenizer''' )




 @require_tokenizers
 def        __magic_name__		( self		:							Any )		->  Optional[int]:






       class        __SCREAMING_SNAKE_CASE (     lowerCamelCase			):
        snake_case_												=							False






       class        __SCREAMING_SNAKE_CASE (     lowerCamelCase			):
        snake_case_												=							NewTokenizer
        snake_case_												=							False

       try:
             AutoConfig.register('''custom'''   , __lowercase )
             AutoTokenizer.register(__lowercase   , slow_tokenizer_class=__lowercase )
             AutoTokenizer.register(__lowercase   , fast_tokenizer_class=__lowercase )
             # If remote code is not set, the default is to use local
             SCREAMING_SNAKE_CASE__       :			Optional[Any]													=AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizerFast''' )
             self.assertFalse(tokenizer.special_attribute_present )
             SCREAMING_SNAKE_CASE__       :			Optional[int]													=AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''   , use_fast=__lowercase )
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizer''' )
             self.assertFalse(tokenizer.special_attribute_present )

             # If remote code is disabled, we load the local one.
             SCREAMING_SNAKE_CASE__       :			List[str]													=AutoTokenizer.from_pretrained(
                 '''hf-internal-testing/test_dynamic_tokenizer'''   , trust_remote_code=__lowercase )
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizerFast''' )
             self.assertFalse(tokenizer.special_attribute_present )
             SCREAMING_SNAKE_CASE__       :			Dict													=AutoTokenizer.from_pretrained(
                 '''hf-internal-testing/test_dynamic_tokenizer'''   , trust_remote_code=__lowercase   , use_fast=__lowercase )
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizer''' )
             self.assertFalse(tokenizer.special_attribute_present )

             # If remote is enabled, we load from the Hub
             SCREAMING_SNAKE_CASE__       :			int													=AutoTokenizer.from_pretrained(
                 '''hf-internal-testing/test_dynamic_tokenizer'''   , trust_remote_code=__lowercase )
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizerFast''' )
             self.assertTrue(tokenizer.special_attribute_present )
             SCREAMING_SNAKE_CASE__       :			Tuple													=AutoTokenizer.from_pretrained(
                 '''hf-internal-testing/test_dynamic_tokenizer'''   , trust_remote_code=__lowercase   , use_fast=__lowercase )
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizer''' )
             self.assertTrue(tokenizer.special_attribute_present )

       finally:
             if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
             if CustomConfig in TOKENIZER_MAPPING._extra_content:
                   del TOKENIZER_MAPPING._extra_content[CustomConfig]




 def        __magic_name__		( self		:							Tuple )		->  Union[str, Any]:
       SCREAMING_SNAKE_CASE__       :			str													=AutoTokenizer.from_pretrained(
           '''hf-internal-testing/test_dynamic_tokenizer_legacy'''   , trust_remote_code=__lowercase )
       self.assertTrue(tokenizer.special_attribute_present )
       if is_tokenizers_available():
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizerFast''' )

             # Test we can also load the slow version
             SCREAMING_SNAKE_CASE__       :			List[str]													=AutoTokenizer.from_pretrained(
                 '''hf-internal-testing/test_dynamic_tokenizer_legacy'''   , trust_remote_code=__lowercase   , use_fast=__lowercase )
             self.assertTrue(tokenizer.special_attribute_present )
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizer''' )
       else:
             self.assertEqual(tokenizer.__class__.__name__   , '''NewTokenizer''' )




 def        __magic_name__		( self		:							List[str] )		->  Any:
       with self.assertRaisesRegex(
           __lowercase   , '''bert-base is not a local folder and is not a valid model identifier''' ):
             SCREAMING_SNAKE_CASE__       :			Optional[int]													=AutoTokenizer.from_pretrained('''bert-base''' )




 def        __magic_name__		( self		:							Optional[int] )		->  Union[str, Any]:
       with self.assertRaisesRegex(
           __lowercase   , r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)''' ):
             SCREAMING_SNAKE_CASE__       :			Dict													=AutoTokenizer.from_pretrained(__lowercase   , revision='''aaaaaa''' )




 def        __magic_name__		( self		:							Union[str, Any] )		->  Union[str, Any]:
       # Make sure we have cached the tokenizer.
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
       with RequestCounter() as counter:
             SCREAMING_SNAKE_CASE__       :			Any													=AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
             self.assertEqual(counter.get_request_count   , 0 )
             self.assertEqual(counter.head_request_count   , 1 )
             self.assertEqual(counter.other_request_count   , 0 )







'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import AutoTokenizer, PegasusConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import TFAutoModelForSeqaSeqLM, TFPegasusForConditionalGeneration, TFPegasusModel






@require_tf
class        __SCREAMING_SNAKE_CASE :
 snake_case_												=							PegasusConfig
 snake_case_												=							{}
 snake_case_												=							"""gelu"""




 def __init__( self		:							int   , __lowercase		:							Optional[Any]   , __lowercase		:							int=13   , __lowercase		:							List[str]=7   , __lowercase		:							Dict=True   , __lowercase		:							Tuple=False   , __lowercase		:							Optional[Any]=99   , __lowercase		:							str=32   , __lowercase		:							List[str]=2   , __lowercase		:							str=4   , __lowercase		:							Optional[int]=37   , __lowercase		:							List[Any]=0.1   , __lowercase		:							List[Any]=0.1   , __lowercase		:							List[Any]=40   , __lowercase		:							str=2   , __lowercase		:							List[Any]=1   , __lowercase		:							Optional[Any]=0   , )		->  Tuple:
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=parent
       SCREAMING_SNAKE_CASE__       :			List[Any]													=batch_size
       SCREAMING_SNAKE_CASE__       :			Optional[int]													=seq_length
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=is_training
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=use_labels
       SCREAMING_SNAKE_CASE__       :			str													=vocab_size
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=hidden_size
       SCREAMING_SNAKE_CASE__       :			List[str]													=num_hidden_layers
       SCREAMING_SNAKE_CASE__       :			int													=num_attention_heads
       SCREAMING_SNAKE_CASE__       :			List[str]													=intermediate_size

       SCREAMING_SNAKE_CASE__       :			List[Any]													=hidden_dropout_prob
       SCREAMING_SNAKE_CASE__       :			List[Any]													=attention_probs_dropout_prob
       SCREAMING_SNAKE_CASE__       :			List[Any]													=max_position_embeddings
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=eos_token_id
       SCREAMING_SNAKE_CASE__       :			Any													=pad_token_id
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=bos_token_id




 def        __magic_name__		( self		:							Any )		->  str:
       SCREAMING_SNAKE_CASE__       :			List[str]													=ids_tensor([self.batch_size, self.seq_length - 1]   , self.vocab_size )
       SCREAMING_SNAKE_CASE__       :			Optional[int]													=tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size )   , 1 )
       SCREAMING_SNAKE_CASE__       :			Any													=tf.concat([input_ids, eos_tensor]   , axis=1 )

       SCREAMING_SNAKE_CASE__       :			Dict													=ids_tensor([self.batch_size, self.seq_length]   , self.vocab_size )

       SCREAMING_SNAKE_CASE__       :			Optional[int]													=self.config_cls(
           vocab_size=self.vocab_size   , d_model=self.hidden_size   , encoder_layers=self.num_hidden_layers   , decoder_layers=self.num_hidden_layers   , encoder_attention_heads=self.num_attention_heads   , decoder_attention_heads=self.num_attention_heads   , encoder_ffn_dim=self.intermediate_size   , decoder_ffn_dim=self.intermediate_size   , dropout=self.hidden_dropout_prob   , attention_dropout=self.attention_probs_dropout_prob   , max_position_embeddings=self.max_position_embeddings   , eos_token_ids=[2]   , bos_token_id=self.bos_token_id   , pad_token_id=self.pad_token_id   , decoder_start_token_id=self.pad_token_id   , **self.config_updates   , )
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=prepare_pegasus_inputs_dict(__lowercase   , __lowercase   , __lowercase )
       return config, inputs_dict




 def        __magic_name__		( self		:							Optional[int]   , __lowercase		:							List[str]   , __lowercase		:							Optional[int] )		->  Tuple:
       SCREAMING_SNAKE_CASE__       :			List[Any]													=TFPegasusModel(config=__lowercase ).get_decoder()
       SCREAMING_SNAKE_CASE__       :			List[str]													=inputs_dict['''input_ids''']

       SCREAMING_SNAKE_CASE__       :			Tuple													=input_ids[:1, :]
       SCREAMING_SNAKE_CASE__       :			Tuple													=inputs_dict['''attention_mask'''][:1, :]
       SCREAMING_SNAKE_CASE__       :			Tuple													=inputs_dict['''head_mask''']
       SCREAMING_SNAKE_CASE__       :			List[str]													=1

       # first forward pass
       SCREAMING_SNAKE_CASE__       :			Any													=model(__lowercase   , attention_mask=__lowercase   , head_mask=__lowercase   , use_cache=__lowercase )

       SCREAMING_SNAKE_CASE__						,		SCREAMING_SNAKE_CASE__       :			Tuple													=outputs.to_tuple()

       # create hypothetical next token and extent to next_input_ids
       SCREAMING_SNAKE_CASE__       :			str													=ids_tensor((self.batch_size, 3)   , config.vocab_size )
       SCREAMING_SNAKE_CASE__       :			List[Any]													=tf.cast(ids_tensor((self.batch_size, 3)   , 2 )   , tf.inta )

       # append to next input_ids and
       SCREAMING_SNAKE_CASE__       :			Tuple													=tf.concat([input_ids, next_tokens]   , axis=-1 )
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=tf.concat([attention_mask, next_attn_mask]   , axis=-1 )

       SCREAMING_SNAKE_CASE__       :			int													=model(__lowercase   , attention_mask=__lowercase )[0]
       SCREAMING_SNAKE_CASE__       :			Any													=model(__lowercase   , attention_mask=__lowercase   , past_key_values=__lowercase )[0]

       self.parent.assertEqual(next_tokens.shape[1]   , output_from_past.shape[1] )

       # select random slice
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=int(ids_tensor((1,)   , output_from_past.shape[-1] ) )
       SCREAMING_SNAKE_CASE__       :			Any													=output_from_no_past[:, -3:, random_slice_idx]
       SCREAMING_SNAKE_CASE__       :			List[str]													=output_from_past[:, :, random_slice_idx]

       # test that outputs are equal for slice
       tf.debugging.assert_near(__lowercase   , __lowercase   , rtol=1e-3 )





def 	_a(					UpperCamelCase__			:  Any,		UpperCamelCase__			:  Union[str, Any],		UpperCamelCase__			:  Union[str, Any],		UpperCamelCase__			:  Optional[Any]=None,		UpperCamelCase__			:  Optional[Any]=None,		UpperCamelCase__			:  Union[str, Any]=None,		UpperCamelCase__			:  Any=None,		UpperCamelCase__			:  Optional[Any]=None,		):




      '''simple docstring'''






      if attention_mask is None:
            SCREAMING_SNAKE_CASE__       :			str													=tf.cast(tf.math.not_equal(UpperCamelCase__,		config.pad_token_id							),		tf.inta							)
      if decoder_attention_mask is None:
            SCREAMING_SNAKE_CASE__       :			Any													=tf.concat(
                [
                    tf.ones(decoder_input_ids[:, :1].shape,		dtype=tf.inta							),
                    tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:],		config.pad_token_id							),		tf.inta							),
                ],		axis=-1,		)
      if head_mask is None:
            SCREAMING_SNAKE_CASE__       :			int													=tf.ones((config.encoder_layers, config.encoder_attention_heads)							)
      if decoder_head_mask is None:
            SCREAMING_SNAKE_CASE__       :			List[Any]													=tf.ones((config.decoder_layers, config.decoder_attention_heads)							)
      if cross_attn_head_mask is None:
            SCREAMING_SNAKE_CASE__       :			List[str]													=tf.ones((config.decoder_layers, config.decoder_attention_heads)							)
      return {
          "input_ids": input_ids,
          "decoder_input_ids": decoder_input_ids,
          "attention_mask": attention_mask,
          "decoder_attention_mask": decoder_attention_mask,
          "head_mask": head_mask,
          "decoder_head_mask": decoder_head_mask,
          "cross_attn_head_mask": cross_attn_head_mask,
      }






@require_tf
class        __SCREAMING_SNAKE_CASE (     lowerCamelCase		,			lowerCamelCase		,			unittest.TestCase			):
 snake_case_												=							(TFPegasusForConditionalGeneration, TFPegasusModel) if is_tf_available() else ()
 snake_case_												=							(TFPegasusForConditionalGeneration,) if is_tf_available() else ()
 snake_case_												=							(
     {
         """conversational""": TFPegasusForConditionalGeneration,
         """feature-extraction""": TFPegasusModel,
         """summarization""": TFPegasusForConditionalGeneration,
         """text2text-generation""": TFPegasusForConditionalGeneration,
         """translation""": TFPegasusForConditionalGeneration,
     }
     if is_tf_available()
     else {}
 )
 snake_case_												=							True
 snake_case_												=							False
 snake_case_												=							False




 def        __magic_name__		( self		:							Union[str, Any] )		->  str:
       SCREAMING_SNAKE_CASE__       :			List[Any]													=TFPegasusModelTester(self )
       SCREAMING_SNAKE_CASE__       :			Dict													=ConfigTester(self   , config_class=__lowercase )




 def        __magic_name__		( self		:							int )		->  Any:
       self.config_tester.run_common_tests()




 def        __magic_name__		( self		:							Optional[Any] )		->  List[Any]:
       SCREAMING_SNAKE_CASE__       :			Tuple													=self.model_tester.prepare_config_and_inputs_for_common()
       self.model_tester.check_decoder_model_past_large_inputs(*__lowercase )






@require_sentencepiece
@require_tokenizers
@require_tf
class        __SCREAMING_SNAKE_CASE (     unittest.TestCase			):
 snake_case_												=							[
     """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.""",
     """ The London trio are up for best UK act and best album, as well as getting two nominations in the best song category.\"We got told like this morning 'Oh I think you're nominated'\", said Dappy.\"And I was like 'Oh yeah, which one?' And now we've got nominated for four awards. I mean, wow!\"Bandmate Fazer added: \"We thought it's best of us to come down and mingle with everyone and say hello to the cameras. And now we find we've got four nominations.\"The band have two shots at the best song prize, getting the nod for their Tynchy Stryder collaboration Number One, and single Strong Again.Their album Uncle B will also go up against records by the likes of Beyonce and Kanye West.N-Dubz picked up the best newcomer Mobo in 2007, but female member Tulisa said they wouldn't be too disappointed if they didn't win this time around.\"At the end of the day we're grateful to be where we are in our careers.\"If it don't happen then it don't happen - live to fight another day and keep on making albums and hits for the fans.\"Dappy also revealed they could be performing live several times on the night.The group will be doing Number One and also a possible rendition of the War Child single, I Got Soul.The charity song is a  re-working of The Killers' All These Things That I've Done and is set to feature artists like Chipmunk, Ironik and Pixie Lott.This year's Mobos will be held outside of London for the first time, in Glasgow on 30 September.N-Dubz said they were looking forward to performing for their Scottish fans and boasted about their recent shows north of the border.\"We just done Edinburgh the other day,\" said Dappy.\"We smashed up an N-Dubz show over there. We done Aberdeen about three or four months ago - we smashed up that show over there! Everywhere we go we smash it up!\" """,
 ]
 snake_case_												=							[
     """California's largest electricity provider has cut power to hundreds of thousands of customers in an effort to"""
     """ reduce the risk of wildfires.""",
     """N-Dubz have revealed they\'re \"grateful\" to have been nominated for four Mobo Awards.""",
 ]  # differs slightly from pytorch, likely due to numerical differences in linear layers
 snake_case_												=							"""google/pegasus-xsum"""




 @cached_property
 def        __magic_name__		( self		:							Optional[int] )		->  Tuple:
       return AutoTokenizer.from_pretrained(self.model_name )




 @cached_property
 def        __magic_name__		( self		:							List[Any] )		->  Optional[Any]:
       SCREAMING_SNAKE_CASE__       :			Optional[int]													=TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name )
       return model




 def        __magic_name__		( self		:							List[str]   , **__lowercase		:							Any )		->  Optional[int]:
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=self.translate_src_text(**__lowercase )
       assert self.expected_text == generated_words




 def        __magic_name__		( self		:							Optional[Any]   , **__lowercase		:							List[str] )		->  Optional[int]:
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=self.tokenizer(self.src_text   , **__lowercase   , padding=__lowercase   , return_tensors='''tf''' )
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=self.model.generate(
           model_inputs.input_ids   , attention_mask=model_inputs.attention_mask   , num_beams=2   , use_cache=__lowercase   , )
       SCREAMING_SNAKE_CASE__       :			Any													=self.tokenizer.batch_decode(generated_ids.numpy()   , skip_special_tokens=__lowercase )
       return generated_words




 @slow
 def        __magic_name__		( self		:							Optional[Any] )		->  Optional[int]:
       self._assert_generated_batch_equal_expected()