






"""simple docstring"""



import warnings
from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


A__	:    List[Any]			 = logging.get_logger(__name__)

A__	:    str			 = {
    'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/config.json',
    # See all BART models at https://huggingface.co/models?filter=bart
}







class        lowercase__      (							snake_case__       ):
			_UpperCAmelCase		:List[str]             =			"bart"
			_UpperCAmelCase		:Optional[Any]             =			["past_key_values"]
			_UpperCAmelCase		:List[str]             =			{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}


			def __init__(				self					:  Tuple       ,	snake_case__					:  Any=5_0265       ,	snake_case__					:  Optional[int]=1024       ,	snake_case__					:  List[Any]=12       ,	snake_case__					:  Dict=4096       ,	snake_case__					:  Any=16       ,	snake_case__					:  Any=12       ,	snake_case__					:  Dict=4096       ,	snake_case__					:  Optional[Any]=16       ,	snake_case__					:  List[str]=0.0       ,	snake_case__					:  List[Any]=0.0       ,	snake_case__					:  Any="gelu"       ,	snake_case__					:  Optional[Any]=1024       ,	snake_case__					:  str=0.1       ,	snake_case__					:  int=0.0       ,	snake_case__					:  Any=0.0       ,	snake_case__					:  Tuple=0.02       ,	snake_case__					:  str=0.0       ,	snake_case__					:  Any=False       ,	snake_case__					:  Optional[int]=True       ,	snake_case__					:  Tuple=3       ,	snake_case__					:  Optional[Any]=1       ,	snake_case__					:  List[str]=0       ,	snake_case__					:  List[str]=2       ,	snake_case__					:  Optional[Any]=True       ,	snake_case__					:  Tuple=2       ,	snake_case__					:  Dict=2       ,	**snake_case__					:  List[Any]       ,	):
										lowerCamelCase_	:							str									=vocab_size
										lowerCamelCase_	:							int									=max_position_embeddings
										lowerCamelCase_	:							Optional[Any]									=d_model
										lowerCamelCase_	:							Any									=encoder_ffn_dim
										lowerCamelCase_	:							Tuple									=encoder_layers
										lowerCamelCase_	:							int									=encoder_attention_heads
										lowerCamelCase_	:							int									=decoder_ffn_dim
										lowerCamelCase_	:							List[Any]									=decoder_layers
										lowerCamelCase_	:							List[str]									=decoder_attention_heads
										lowerCamelCase_	:							Dict									=dropout
										lowerCamelCase_	:							Optional[Any]									=attention_dropout
										lowerCamelCase_	:							Optional[Any]									=activation_dropout
										lowerCamelCase_	:							Optional[int]									=activation_function
										lowerCamelCase_	:							Union[str, Any]									=init_std
										lowerCamelCase_	:							Any									=encoder_layerdrop
										lowerCamelCase_	:							str									=decoder_layerdrop
										lowerCamelCase_	:							List[str]									=classifier_dropout
										lowerCamelCase_	:							int									=use_cache
										lowerCamelCase_	:							List[Any]									=encoder_layers
										lowerCamelCase_	:							List[Any]									=scale_embedding  # scale factor will be sqrt(d_model) if True

										super().__init__(
										    num_labels=snake_case__       ,	pad_token_id=snake_case__       ,	bos_token_id=snake_case__       ,	eos_token_id=snake_case__       ,	is_encoder_decoder=snake_case__       ,	decoder_start_token_id=snake_case__       ,	forced_eos_token_id=snake_case__       ,	**snake_case__       ,	)

										# ensure backward compatibility for BART CNN models
										if self.forced_bos_token_id is None and kwargs.get("force_bos_token_to_be_generated"       ,	snake_case__ ):
																	lowerCamelCase_	:							str									=self.bos_token_id
																	warnings.warn(
																	    F"""Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. """
																	    "The config can simply be saved and uploaded again to be fixed." )







class        lowercase__      (							snake_case__       ):


			@property
			def      UpperCAmelCase__						(				self					:  Optional[Any] ):
										if self.task in ["default", "seq2seq-lm"]:
																	lowerCamelCase_	:							List[str]									=OrderedDict(
																	    [
																	        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
																	        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
																	    ] )

																	if self.use_past:
																								lowerCamelCase_	:							Union[str, Any]									={0: "batch"}
																								lowerCamelCase_	:							str									={0: "batch", 1: "past_decoder_sequence + sequence"}
																	else:
																								lowerCamelCase_	:							List[Any]									={0: "batch", 1: "decoder_sequence"}
																								lowerCamelCase_	:							Optional[Any]									={0: "batch", 1: "decoder_sequence"}

																	if self.use_past:
																								self.fill_with_past_key_values_(snake_case__       ,	direction="inputs" )
										elif self.task == "causal-lm":
																	# TODO: figure this case out.
																	lowerCamelCase_	:							List[str]									=OrderedDict(
																	    [
																	        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
																	        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
																	    ] )
																	if self.use_past:
																								lowerCamelCase_							,						lowerCamelCase_	:							List[Any]									=self.num_layers
																								for i in range(snake_case__ ):
																															lowerCamelCase_	:							int									={0: "batch", 2: "past_sequence + sequence"}
																															lowerCamelCase_	:							Dict									={0: "batch", 2: "past_sequence + sequence"}
										else:
																	lowerCamelCase_	:							Tuple									=OrderedDict(
																	    [
																	        ("input_ids", {0: "batch", 1: "encoder_sequence"}),
																	        ("attention_mask", {0: "batch", 1: "encoder_sequence"}),
																	        ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}),
																	        ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}),
																	    ] )

										return common_inputs


			@property
			def      UpperCAmelCase__						(				self					:  Tuple ):
										if self.task in ["default", "seq2seq-lm"]:
																	lowerCamelCase_	:							Optional[Any]									=super().outputs
										else:
																	lowerCamelCase_	:							Union[str, Any]									=super(snake_case__       ,	self ).outputs
																	if self.use_past:
																								lowerCamelCase_							,						lowerCamelCase_	:							List[Any]									=self.num_layers
																								for i in range(snake_case__ ):
																															lowerCamelCase_	:							Optional[int]									={0: "batch", 2: "past_sequence + sequence"}
																															lowerCamelCase_	:							List[Any]									={0: "batch", 2: "past_sequence + sequence"}
										return common_outputs


			def      UpperCAmelCase__						(				self					:  str       ,	snake_case__					:  PreTrainedTokenizer       ,	snake_case__					:  int = -1       ,	snake_case__					:  int = -1       ,	snake_case__					:  bool = False       ,	snake_case__					:  Optional[TensorType] = None       ,	):
										lowerCamelCase_	:							Dict									=self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    snake_case__       ,	snake_case__       ,	snake_case__       ,	snake_case__       ,	snake_case__ )

										# Generate decoder inputs
										lowerCamelCase_	:							int									=seq_length if not self.use_past else 1
										lowerCamelCase_	:							str									=self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    snake_case__       ,	snake_case__       ,	snake_case__       ,	snake_case__       ,	snake_case__ )
										lowerCamelCase_	:							List[Any]									={F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
										lowerCamelCase_	:							List[str]									=dict(**snake_case__       ,	**snake_case__ )

										if self.use_past:
																	if not is_torch_available():
																								raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed." )
																	else:
																								import torch
																	lowerCamelCase_							,						lowerCamelCase_	:							int									=common_inputs["input_ids"].shape
																	lowerCamelCase_	:							Any									=common_inputs["decoder_input_ids"].shape[1]
																	lowerCamelCase_							,						lowerCamelCase_	:							Tuple									=self.num_attention_heads
																	lowerCamelCase_	:							str									=(
																	    batch,
																	    num_encoder_attention_heads,
																	    encoder_seq_length,
																	    self._config.hidden_size // num_encoder_attention_heads,
																	)
																	lowerCamelCase_	:							List[Any]									=decoder_seq_length + 3
																	lowerCamelCase_	:							Optional[int]									=(
																	    batch,
																	    num_decoder_attention_heads,
																	    decoder_past_length,
																	    self._config.hidden_size // num_decoder_attention_heads,
																	)

																	lowerCamelCase_	:							Tuple									=torch.cat(
																	    [common_inputs["decoder_attention_mask"], torch.ones(snake_case__       ,	snake_case__ )]       ,	dim=1 )

																	lowerCamelCase_	:							List[str]									=[]
																	# If the number of encoder and decoder layers are present in the model configuration, both are considered
																	lowerCamelCase_							,						lowerCamelCase_	:							Optional[Any]									=self.num_layers
																	lowerCamelCase_	:							Optional[int]									=min(snake_case__       ,	snake_case__ )
																	lowerCamelCase_	:							Optional[Any]									=max(snake_case__       ,	snake_case__ ) - min_num_layers
																	lowerCamelCase_	:							Any									="encoder" if num_encoder_layers > num_decoder_layers else "decoder"

																	for _ in range(snake_case__ ):
																								common_inputs["past_key_values"].append(
																								    (
																								        torch.zeros(snake_case__ ),
																								        torch.zeros(snake_case__ ),
																								        torch.zeros(snake_case__ ),
																								        torch.zeros(snake_case__ ),
																								    ) )
																	# TODO: test this.
																	lowerCamelCase_	:							List[Any]									=encoder_shape if remaining_side_name == "encoder" else decoder_shape
																	for _ in range(snake_case__       ,	snake_case__ ):
																								common_inputs["past_key_values"].append((torch.zeros(snake_case__ ), torch.zeros(snake_case__ )) )
										return common_inputs


			def      UpperCAmelCase__						(				self					:  Any       ,	snake_case__					:  PreTrainedTokenizer       ,	snake_case__					:  int = -1       ,	snake_case__					:  int = -1       ,	snake_case__					:  bool = False       ,	snake_case__					:  Optional[TensorType] = None       ,	):
										lowerCamelCase_	:							str									=self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
										    snake_case__       ,	snake_case__       ,	snake_case__       ,	snake_case__       ,	snake_case__ )

										if self.use_past:
																	if not is_torch_available():
																								raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed." )
																	else:
																								import torch
																	lowerCamelCase_							,						lowerCamelCase_	:							Tuple									=common_inputs["input_ids"].shape
																	# Not using the same length for past_key_values
																	lowerCamelCase_	:							str									=seqlen + 2
																	lowerCamelCase_							,						lowerCamelCase_	:							List[Any]									=self.num_layers
																	lowerCamelCase_							,						lowerCamelCase_	:							List[Any]									=self.num_attention_heads
																	lowerCamelCase_	:							List[Any]									=(
																	    batch,
																	    num_encoder_attention_heads,
																	    past_key_values_length,
																	    self._config.hidden_size // num_encoder_attention_heads,
																	)

																	lowerCamelCase_	:							List[Any]									=common_inputs["attention_mask"].dtype
																	lowerCamelCase_	:							Tuple									=torch.cat(
																	    [common_inputs["attention_mask"], torch.ones(snake_case__       ,	snake_case__       ,	dtype=snake_case__ )]       ,	dim=1 )
																	lowerCamelCase_	:							Dict									=[
																	    (torch.zeros(snake_case__ ), torch.zeros(snake_case__ )) for _ in range(snake_case__ )
																	]
										return common_inputs


			def      UpperCAmelCase__						(				self					:  Union[str, Any]       ,	snake_case__					:  PreTrainedTokenizer       ,	snake_case__					:  int = -1       ,	snake_case__					:  int = -1       ,	snake_case__					:  bool = False       ,	snake_case__					:  Optional[TensorType] = None       ,	):
										# Copied from OnnxConfig.generate_dummy_inputs
										# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
										# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
										lowerCamelCase_	:							Dict									=compute_effective_axis_dimension(
										    snake_case__       ,	fixed_dimension=OnnxConfig.default_fixed_batch       ,	num_token_to_add=0 )

										# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
										lowerCamelCase_	:							int									=tokenizer.num_special_tokens_to_add(snake_case__ )
										lowerCamelCase_	:							Optional[int]									=compute_effective_axis_dimension(
										    snake_case__       ,	fixed_dimension=OnnxConfig.default_fixed_sequence       ,	num_token_to_add=snake_case__ )

										# Generate dummy inputs according to compute batch and sequence
										lowerCamelCase_	:							int									=[" ".join([tokenizer.unk_token] ) * seq_length] * batch_size
										lowerCamelCase_	:							Tuple									=dict(tokenizer(snake_case__       ,	return_tensors=snake_case__ ) )
										return common_inputs


			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  PreTrainedTokenizer       ,	snake_case__					:  int = -1       ,	snake_case__					:  int = -1       ,	snake_case__					:  bool = False       ,	snake_case__					:  Optional[TensorType] = None       ,	):
										if self.task in ["default", "seq2seq-lm"]:
																	lowerCamelCase_	:							str									=self._generate_dummy_inputs_for_default_and_seqaseq_lm(
																	    snake_case__       ,	batch_size=snake_case__       ,	seq_length=snake_case__       ,	is_pair=snake_case__       ,	framework=snake_case__ )

										elif self.task == "causal-lm":
																	lowerCamelCase_	:							Optional[int]									=self._generate_dummy_inputs_for_causal_lm(
																	    snake_case__       ,	batch_size=snake_case__       ,	seq_length=snake_case__       ,	is_pair=snake_case__       ,	framework=snake_case__ )
										else:
																	lowerCamelCase_	:							int									=self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
																	    snake_case__       ,	batch_size=snake_case__       ,	seq_length=snake_case__       ,	is_pair=snake_case__       ,	framework=snake_case__ )

										return common_inputs


			def      UpperCAmelCase__						(				self					:  List[str]       ,	snake_case__					:  int       ,	snake_case__					:  Optional[int]       ,	snake_case__					:  str       ,	snake_case__					:  Optional[int] ):
										if self.task in ["default", "seq2seq-lm"]:
																	lowerCamelCase_	:							Optional[int]									=super()._flatten_past_key_values_(snake_case__       ,	snake_case__       ,	snake_case__       ,	snake_case__ )
										else:
																	lowerCamelCase_	:							Optional[Any]									=super(snake_case__       ,	self )._flatten_past_key_values_(
																	    snake_case__       ,	snake_case__       ,	snake_case__       ,	snake_case__ )











"""simple docstring"""



from typing import List, Optional

from tokenizers import ByteLevelBPETokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot_small import BlenderbotSmallTokenizer


A__	:    Dict			 = logging.get_logger(__name__)

A__	:    Dict			 = {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

A__	:    List[Any]			 = {
    'vocab_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json'
    },
    'merges_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt'
    },
    'tokenizer_config_file': {
        'facebook/blenderbot_small-90M': (
            'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json'
        )
    },
}

A__	:    Optional[int]			 = {
    'facebook/blenderbot_small-90M': 512,
}







class        lowercase__      (							snake_case__       ):
			_UpperCAmelCase		:Optional[int]             =			VOCAB_FILES_NAMES
			_UpperCAmelCase		:Tuple             =			PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase		:Any             =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase		:Tuple             =			BlenderbotSmallTokenizer


			def __init__(				self					:  Tuple       ,	snake_case__					:  Optional[Any]=None       ,	snake_case__					:  str=None       ,	snake_case__					:  Any="<|endoftext|>"       ,	snake_case__					:  Tuple="<|endoftext|>"       ,	snake_case__					:  Tuple="<|endoftext|>"       ,	snake_case__					:  str=False       ,	snake_case__					:  int=True       ,	**snake_case__					:  Tuple       ,	):
										super().__init__(
										    ByteLevelBPETokenizer(
										        vocab=snake_case__       ,	merges=snake_case__       ,	add_prefix_space=snake_case__       ,	trim_offsets=snake_case__       ,	)       ,	bos_token=snake_case__       ,	eos_token=snake_case__       ,	unk_token=snake_case__       ,	**snake_case__       ,	)
										lowerCamelCase_	:							Optional[int]									=add_prefix_space


			def      UpperCAmelCase__						(				self					:  Tuple       ,	snake_case__					:  Optional[Any]       ,	snake_case__					:  List[str]=None ):
										lowerCamelCase_	:							Optional[Any]									=[self.bos_token_id] + token_ids_a + [self.eos_token_id]
										if token_ids_a is None:
																	return output

										return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


			def      UpperCAmelCase__						(				self					:  Tuple       ,	snake_case__					:  List[int]       ,	snake_case__					:  Optional[List[int]] = None ):
										lowerCamelCase_	:							int									=[self.sep_token_id]
										lowerCamelCase_	:							List[Any]									=[self.cls_token_id]

										if token_ids_a is None:
																	return len(cls + token_ids_a + sep ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]



