from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import (
							    AutoTokenizer,
							    TFAutoModelForCausalLM,
							    TFAutoModelForSeqaSeqLM,
							    TFAutoModelForSpeechSeqaSeq,
							    TFAutoModelForVisionaSeq,
							    TFBartForConditionalGeneration,
							    TFLogitsProcessorList,
							    TFMinLengthLogitsProcessor,
							    tf_top_k_top_p_filtering,
							)

if is_tensorflow_text_available():
							import tensorflow_text as text


@require_tf
class   lowercase      (       unittest.TestCase       ):




							def 		__snake_case(  self				:   Tuple )   ->			List[str]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE           =    tf.convert_to_tensor(
												    [
												        [
												            8.2_2_2_0_9_9_1,  # 3rd highest value; idx. 0
												            -0.5_6_2_0_0_4_4,
												            5.2_3_2_2_9_7_5_2,
												            4.0_3_8_6_3_9_3,
												            -6.8_7_9_8_3_7_8,
												            -0.5_4_7_8_5_8_0_2,
												            -3.2_0_1_2_1_5_3,
												            2.9_2_7_7_7_1_7_6,
												            1.8_8_1_7_1_9_5_3,
												            7.3_5_3_4_1_2_7_6,  # 5th highest value; idx. 9
												            8.4_3_2_0_7_8_3_3,  # 2nd highest value; idx. 10
												            -9.8_5_7_1_1_8_3_6,
												            -5.9_6_2_0_9_2_3_6,
												            -1.1_3_0_3_9_1_6_1,
												            -7.1_1_1_5_2_9_4,
												            -0.8_3_6_9_6_3_3,
												            -5.3_1_8_6_4_0_8,
												            7.0_6_4_2_7_4_0_7,
												            0.8_1_3_6_9_3_4_4,
												            -0.8_2_0_2_3_8_1_7,
												            -5.9_1_7_9_7_9_6,
												            0.5_8_8_1_3_4_4_3,
												            -6.9_9_7_7_8_4_3_8,
												            4.7_1_5_5_1_1_8_9,
												            -0.1_8_7_7_1_6_3_7,
												            7.4_4_0_2_0_7_5_9,  # 4th highest value; idx. 25
												            9.3_8_4_5_0_9_8_7,  # 1st highest value; idx. 26
												            2.1_2_6_6_2_9_4_1,
												            -9.3_2_5_6_2_0_3_8,
												            2.3_5_6_5_2_5_2_2,
												        ],  # cummulative prob of 5 highest values <= 0.6
												        [
												            0.5_8_4_2_5_5_1_8,
												            4.5_3_1_3_9_2_3_8,
												            -5.5_7_5_1_0_4_6_4,
												            -6.2_8_0_3_0_6_9_9,
												            -7.1_9_5_2_9_5_0_3,
												            -4.0_2_1_2_2_5_5_1,
												            1.3_9_3_3_7_0_3_7,
												            -6.0_6_7_0_7_0_5_7,
												            1.5_9_4_8_0_5_1_7,
												            -9.6_4_3_1_1_9,
												            0.0_3_9_0_7_7_9_9,
												            0.6_7_2_3_1_7_6_2,
												            -8.8_8_2_0_6_7_2_6,
												            6.2_7_1_1_5_9_2_2,  # 4th highest value; idx. 13
												            2.2_8_5_2_0_7_2_3,
												            4.8_2_7_6_7_5_0_6,
												            4.3_0_4_2_1_3_6_8,
												            8.8_2_7_5_3_1_3,  # 2nd highest value; idx. 17
												            5.4_4_0_2_9_9_5_8,  # 5th highest value; idx. 18
												            -4.4_7_3_5_7_9_4,
												            7.3_8_5_7_9_5_3_6,  # 3rd highest value; idx. 20
												            -2.9_1_0_5_1_6_6_3,
												            2.6_1_9_4_6_0_7_7,
												            -2.5_6_7_4_7_6_2,
												            -9.4_8_9_5_9_3_0_2,
												            -4.0_2_9_2_2_6_4_5,
												            -1.3_5_4_1_6_9_1_8,
												            9.6_7_7_0_2_3_2_3,  # 1st highest value; idx. 27
												            -5.8_9_4_7_8_5_5_3,
												            1.8_5_3_7_0_4_6_7,
												        ],  # cummulative prob of 5 highest values <= 0.6
												    ]		,							dtype=tf.floataa		,							)

												SCREAMING_SNAKE_CASE           =    tf.convert_to_tensor(
												    [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]]		,							dtype=tf.intaa		,							)  # expected non filtered idx as noted above

												SCREAMING_SNAKE_CASE           =    tf.convert_to_tensor(
												    [8.2_2_2_0_9_9, 7.3_5_3_4_1_2_6, 8.4_3_2_0_7_8, 7.4_4_0_2_0_7_5, 9.3_8_4_5_1, 6.2_7_1_1_5_9, 8.8_2_7_5_3_1, 5.4_4_0_2_9_9_5, 7.3_8_5_7_9_5_6, 9.6_7_7_0_2_3]		,							dtype=tf.floataa		,							)  # expected non filtered values as noted above

												SCREAMING_SNAKE_CASE           =    tf_top_k_top_p_filtering(_UpperCamelCase		,							top_k=10		,							top_p=0.6		,							min_tokens_to_keep=4 )

												SCREAMING_SNAKE_CASE           =    output[output != -float("inf" )]
												SCREAMING_SNAKE_CASE           =    tf.cast(
												    tf.where(tf.not_equal(_UpperCamelCase		,							tf.constant(-float("inf" )		,							dtype=tf.floataa ) ) )		,							dtype=tf.intaa		,							)

												tf.debugging.assert_near(_UpperCamelCase		,							_UpperCamelCase		,							rtol=1e-12 )
												tf.debugging.assert_equal(_UpperCamelCase		,							_UpperCamelCase )



@require_tf
class   lowercase      (       unittest.TestCase			,    a       ):
							# setting framework_dependent_parameters needs to be gated, just like its contents' imports
							if is_tf_available():
														lowercase__  :    List[Any]									=							{
														    """AutoModelForCausalLM""": TFAutoModelForCausalLM,
														    """AutoModelForSpeechSeq2Seq""": TFAutoModelForSpeechSeqaSeq,
														    """AutoModelForSeq2SeqLM""": TFAutoModelForSeqaSeqLM,
														    """AutoModelForVision2Seq""": TFAutoModelForVisionaSeq,
														    """LogitsProcessorList""": TFLogitsProcessorList,
														    """MinLengthLogitsProcessor""": TFMinLengthLogitsProcessor,
														    """create_tensor_fn""": tf.convert_to_tensor,
														    """floats_tensor""": floats_tensor,
														    """return_tensors""": """tf""",
														}




							@slow
							def 		__snake_case(  self				:   List[str] )   ->			Optional[int]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE           =    TFAutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2" )
												SCREAMING_SNAKE_CASE           =    2
												SCREAMING_SNAKE_CASE           =    2


												class   lowercase      (       tf.Module       ):




																			def __init__(  self				:   Optional[Any]		,							_UpperCamelCase				:   Optional[Any] )   ->			int:



																								'''simple docstring'''

																								super(_UpperCamelCase		,							self ).__init__()
																								SCREAMING_SNAKE_CASE           =    model





																			@tf.function(
																			    input_signature=(
																			        tf.TensorSpec((None, input_length)		,							tf.intaa		,							name="input_ids" ),
																			        tf.TensorSpec((None, input_length)		,							tf.intaa		,							name="attention_mask" ),
																			    )		,							jit_compile=_UpperCamelCase		,							)
																			def 		__snake_case(  self				:   List[str]		,							_UpperCamelCase				:   Any		,							_UpperCamelCase				:   Tuple )   ->			Union[str, Any]:



																								'''simple docstring'''

																								SCREAMING_SNAKE_CASE           =    self.model.generate(
																								    input_ids=_UpperCamelCase		,							attention_mask=_UpperCamelCase		,							max_new_tokens=_UpperCamelCase		,							return_dict_in_generate=_UpperCamelCase		,							)
																								return {"sequences": outputs["sequences"]}



												SCREAMING_SNAKE_CASE           =    [[2, 0], [102, 103]]
												SCREAMING_SNAKE_CASE           =    [[1, 0], [1, 1]]
												SCREAMING_SNAKE_CASE           =    DummyModel(model=_UpperCamelCase )
												with tempfile.TemporaryDirectory() as tmp_dir:
																	tf.saved_model.save(_UpperCamelCase		,							_UpperCamelCase		,							signatures={"serving_default": dummy_model.serving} )
																	SCREAMING_SNAKE_CASE           =    tf.saved_model.load(_UpperCamelCase ).signatures["serving_default"]
																	for batch_size in range(1		,							len(_UpperCamelCase ) + 1 ):
																						SCREAMING_SNAKE_CASE           =    {
																						    "input_ids": tf.constant(dummy_input_ids[:batch_size] ),
																						    "attention_mask": tf.constant(dummy_attention_masks[:batch_size] ),
																						}
																						SCREAMING_SNAKE_CASE           =    serving_func(**_UpperCamelCase )["sequences"]
																						SCREAMING_SNAKE_CASE           =    test_model.generate(**_UpperCamelCase		,							max_new_tokens=_UpperCamelCase )
																						tf.debugging.assert_equal(_UpperCamelCase		,							_UpperCamelCase )




							@slow
							def 		__snake_case(  self				:   Dict )   ->			Dict:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE           =    TFAutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2" )
												SCREAMING_SNAKE_CASE           =    1
												SCREAMING_SNAKE_CASE           =    2


												class   lowercase      (       tf.Module       ):




																			def __init__(  self				:   Any		,							_UpperCamelCase				:   Optional[Any] )   ->			List[str]:



																								'''simple docstring'''

																								super(_UpperCamelCase		,							self ).__init__()
																								SCREAMING_SNAKE_CASE           =    model





																			@tf.function(
																			    input_signature=(
																			        tf.TensorSpec((batch_size, None)		,							tf.intaa		,							name="input_ids" ),
																			        tf.TensorSpec((batch_size, None)		,							tf.intaa		,							name="attention_mask" ),
																			    )		,							jit_compile=_UpperCamelCase		,							)
																			def 		__snake_case(  self				:   List[str]		,							_UpperCamelCase				:   List[str]		,							_UpperCamelCase				:   List[str] )   ->			Optional[Any]:



																								'''simple docstring'''

																								SCREAMING_SNAKE_CASE           =    self.model.generate(
																								    input_ids=_UpperCamelCase		,							attention_mask=_UpperCamelCase		,							max_new_tokens=_UpperCamelCase		,							return_dict_in_generate=_UpperCamelCase		,							)
																								return {"sequences": outputs["sequences"]}



												SCREAMING_SNAKE_CASE           =    [[2], [102, 103]]
												SCREAMING_SNAKE_CASE           =    [[1], [1, 1]]
												SCREAMING_SNAKE_CASE           =    DummyModel(model=_UpperCamelCase )
												with tempfile.TemporaryDirectory() as tmp_dir:
																	tf.saved_model.save(_UpperCamelCase		,							_UpperCamelCase		,							signatures={"serving_default": dummy_model.serving} )
																	SCREAMING_SNAKE_CASE           =    tf.saved_model.load(_UpperCamelCase ).signatures["serving_default"]
																	for input_row in range(len(_UpperCamelCase ) ):
																						SCREAMING_SNAKE_CASE           =    {
																						    "input_ids": tf.constant([dummy_input_ids[input_row]] ),
																						    "attention_mask": tf.constant([dummy_attention_masks[input_row]] ),
																						}
																						SCREAMING_SNAKE_CASE           =    serving_func(**_UpperCamelCase )["sequences"]
																						SCREAMING_SNAKE_CASE           =    test_model.generate(**_UpperCamelCase		,							max_new_tokens=_UpperCamelCase )
																						tf.debugging.assert_equal(_UpperCamelCase		,							_UpperCamelCase )




							@slow
							@require_tensorflow_text
							def 		__snake_case(  self				:   Any )   ->			str:



												'''simple docstring'''

												with tempfile.TemporaryDirectory() as tmp_dir:
																	# file needed to load the TF tokenizer
																	hf_hub_download(repo_id="google/flan-t5-small"		,							filename="spiece.model"		,							local_dir=_UpperCamelCase )


																	class   lowercase      (       tf.keras.layers.Layer       ):




																								def __init__(  self				:   Dict )   ->			Optional[Any]:



																													'''simple docstring'''

																													super().__init__()
																													SCREAMING_SNAKE_CASE           =    text.SentencepieceTokenizer(
																													    model=tf.io.gfile.GFile(os.path.join(_UpperCamelCase		,							"spiece.model" )		,							"rb" ).read() )
																													SCREAMING_SNAKE_CASE           =    TFAutoModelForSeqaSeqLM.from_pretrained("hf-internal-testing/tiny-random-t5" )





																								def 		__snake_case(  self				:   str		,							_UpperCamelCase				:   Any		,							*_UpperCamelCase				:   str		,							**_UpperCamelCase				:   List[Any] )   ->			List[Any]:



																													'''simple docstring'''

																													SCREAMING_SNAKE_CASE           =    self.tokenizer.tokenize(_UpperCamelCase )
																													SCREAMING_SNAKE_CASE     ,				SCREAMING_SNAKE_CASE           =    text.pad_model_inputs(
																													    _UpperCamelCase		,							max_seq_length=64		,							pad_value=self.model.config.pad_token_id )
																													SCREAMING_SNAKE_CASE           =    self.model.generate(input_ids=_UpperCamelCase		,							attention_mask=_UpperCamelCase )
																													return self.tokenizer.detokenize(_UpperCamelCase )



																	SCREAMING_SNAKE_CASE           =    CompleteSentenceTransformer()
																	SCREAMING_SNAKE_CASE           =    tf.keras.layers.Input(shape=(1,)		,							dtype=tf.string		,							name="inputs" )
																	SCREAMING_SNAKE_CASE           =    complete_model(_UpperCamelCase )
																	SCREAMING_SNAKE_CASE           =    tf.keras.Model(_UpperCamelCase		,							_UpperCamelCase )
																	keras_model.save(_UpperCamelCase )




							def 		__snake_case(  self				:   List[str] )   ->			List[Any]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE           =    {
												    "do_sample": True,
												    "num_beams": 1,
												    "top_p": 0.7,
												    "top_k": 10,
												    "temperature": 0.7,
												}
												SCREAMING_SNAKE_CASE           =    14

												SCREAMING_SNAKE_CASE           =    AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2" )
												SCREAMING_SNAKE_CASE           =    "Hello, my dog is cute and"
												SCREAMING_SNAKE_CASE           =    tokenizer(_UpperCamelCase		,							return_tensors="tf" )
												SCREAMING_SNAKE_CASE           =    TFAutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2" )

												SCREAMING_SNAKE_CASE           =    638
												# forces the generation to happen on CPU, to avoid GPU-related quirks
												with tf.device(":/CPU:0" ):
																	tf.random.set_seed(0 )
																	SCREAMING_SNAKE_CASE           =    model.generate(**_UpperCamelCase		,							eos_token_id=_UpperCamelCase		,							**_UpperCamelCase )
												self.assertTrue(expectation == len(generated_tokens[0] ) )

												SCREAMING_SNAKE_CASE           =    [638, 198]
												with tf.device(":/CPU:0" ):
																	tf.random.set_seed(0 )
																	SCREAMING_SNAKE_CASE           =    model.generate(**_UpperCamelCase		,							eos_token_id=_UpperCamelCase		,							**_UpperCamelCase )
												self.assertTrue(expectation == len(generated_tokens[0] ) )





							def 		__snake_case(  self				:   Optional[Any] )   ->			List[str]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE           =    AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bart" )
												SCREAMING_SNAKE_CASE           =    "Hugging Face is a technology company based in New York and Paris."
												SCREAMING_SNAKE_CASE           =    bart_tokenizer(_UpperCamelCase		,							return_tensors="tf" ).input_ids
												SCREAMING_SNAKE_CASE           =    TFBartForConditionalGeneration.from_pretrained("hf-internal-testing/tiny-random-bart" )
												SCREAMING_SNAKE_CASE           =    bart_model.generate(_UpperCamelCase ).numpy()


												class   lowercase      (       a       ):




																			def 		__snake_case(  self				:   List[Any]		,							_UpperCamelCase				:   Dict		,							_UpperCamelCase				:   Optional[Any]=None		,							**_UpperCamelCase				:   List[Any] )   ->			Dict:



																								'''simple docstring'''

																								return super().call(_UpperCamelCase		,							**_UpperCamelCase )



												SCREAMING_SNAKE_CASE           =    FakeBart.from_pretrained("hf-internal-testing/tiny-random-bart" )
												SCREAMING_SNAKE_CASE           =    bart_model.generate(_UpperCamelCase		,							foo="bar" ).numpy()
												self.assertTrue(np.array_equal(_UpperCamelCase		,							_UpperCamelCase ) )


												class   lowercase      (       bart_model.model.encoder.__class__       ):




																			def 		__snake_case(  self				:   Any		,							_UpperCamelCase				:   Union[str, Any]		,							**_UpperCamelCase				:   Optional[Any] )   ->			Union[str, Any]:



																								'''simple docstring'''

																								return super().call(_UpperCamelCase		,							**_UpperCamelCase )



												SCREAMING_SNAKE_CASE           =    FakeEncoder(bart_model.config		,							bart_model.model.shared )
												SCREAMING_SNAKE_CASE           =    fake_encoder

												# Normal generation still works (the output will be different because the encoder weights are different)
												SCREAMING_SNAKE_CASE           =    bart_model.generate(_UpperCamelCase ).numpy()
												with self.assertRaises(_UpperCamelCase ):
																	# FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
																	bart_model.generate(_UpperCamelCase		,							foo="bar" )





from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


_lowerCamelCase   :				Optional[Any]     			=						logging.get_logger(__name__)

_lowerCamelCase   :				Any     			=						{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowerCamelCase   :				int     			=						{
    '''vocab_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'''
    },
    '''merges_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'''
    },
}

_lowerCamelCase   :				Tuple     			=						{'''allegro/herbert-base-cased''': 5_14}
_lowerCamelCase   :				Optional[int]     			=						{}


class   lowercase      (       a       ):
							lowercase__  :    List[str]									=							VOCAB_FILES_NAMES
							lowercase__  :    str									=							PRETRAINED_VOCAB_FILES_MAP
							lowercase__  :    Tuple									=							PRETRAINED_INIT_CONFIGURATION
							lowercase__  :    Optional[int]									=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__  :    str									=							HerbertTokenizer




							def __init__(  self				:   Dict		,							_UpperCamelCase				:   Any=None		,							_UpperCamelCase				:   Any=None		,							_UpperCamelCase				:   Optional[int]=None		,							_UpperCamelCase				:   Optional[int]="<s>"		,							_UpperCamelCase				:   Union[str, Any]="<unk>"		,							_UpperCamelCase				:   List[str]="<pad>"		,							_UpperCamelCase				:   List[str]="<mask>"		,							_UpperCamelCase				:   Tuple="</s>"		,							**_UpperCamelCase				:   Any		,							)   ->			str:



												'''simple docstring'''

												super().__init__(
												    _UpperCamelCase		,							_UpperCamelCase		,							tokenizer_file=_UpperCamelCase		,							cls_token=_UpperCamelCase		,							unk_token=_UpperCamelCase		,							pad_token=_UpperCamelCase		,							mask_token=_UpperCamelCase		,							sep_token=_UpperCamelCase		,							**_UpperCamelCase		,							)




							def 		__snake_case(  self				:   Optional[Any]		,							_UpperCamelCase				:   List[int]		,							_UpperCamelCase				:   Optional[List[int]] = None )   ->			List[int]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE           =    [self.cls_token_id]
												SCREAMING_SNAKE_CASE           =    [self.sep_token_id]
												if token_ids_a is None:
																	return cls + token_ids_a + sep

												return cls + token_ids_a + sep + token_ids_a + sep




							def 		__snake_case(  self				:   Any		,							_UpperCamelCase				:   List[int]		,							_UpperCamelCase				:   Optional[List[int]] = None		,							_UpperCamelCase				:   bool = False )   ->			List[int]:



												'''simple docstring'''

												if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=_UpperCamelCase		,							token_ids_a=_UpperCamelCase		,							already_has_special_tokens=_UpperCamelCase )

												if token_ids_a is None:
																	return [1] + ([0] * len(_UpperCamelCase )) + [1]
												return [1] + ([0] * len(_UpperCamelCase )) + [1] + ([0] * len(_UpperCamelCase )) + [1]




							def 		__snake_case(  self				:   Union[str, Any]		,							_UpperCamelCase				:   List[int]		,							_UpperCamelCase				:   Optional[List[int]] = None )   ->			List[int]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE           =    [self.sep_token_id]
												SCREAMING_SNAKE_CASE           =    [self.cls_token_id]

												if token_ids_a is None:
																	return len(cls + token_ids_a + sep ) * [0]
												return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]





							def 		__snake_case(  self				:   str		,							_UpperCamelCase				:   str		,							_UpperCamelCase				:   Optional[str] = None )   ->			Tuple[str]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE           =    self._tokenizer.model.save(_UpperCamelCase		,							name=_UpperCamelCase )
												return tuple(_UpperCamelCase )




