
"""simple docstring"""

import enum
import warnings

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
			import tensorflow as tf

			from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

if is_torch_available():
			from ..models.auto.modeling_auto import MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

_A						=			logging.get_logger(__name__)






class 	_lowercase       (    enum.Enum		):
	lowercase_				=     0
	lowercase_				=     1






@add_end_docstrings(__UpperCAmelCase		)
class 	_lowercase       (    __UpperCAmelCase		):
	lowercase_				=     'generated'




	def __init__(			self		,			*UpperCAmelCase_		,			**UpperCAmelCase_ )		->							Union[str, Any]:
			super().__init__(*UpperCAmelCase_		,			**UpperCAmelCase_ )

			self.check_model_type(
			    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
			    if self.framework == 'tf'
			    else MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			**UpperCAmelCase_		,			)		->							Optional[Any]:
			lowerCamelCase					:      Tuple          =   {}
			if truncation is not None:
					lowerCamelCase					:      Tuple          =   truncation

			lowerCamelCase					:      Union[str, Any]          =   generate_kwargs

			lowerCamelCase					:      List[str]          =   {}
			if return_tensors is not None and return_type is None:
					lowerCamelCase					:      int          =   ReturnType.TENSORS if return_tensors else ReturnType.TEXT
			if return_type is not None:
					lowerCamelCase					:      Optional[int]          =   return_type

			if clean_up_tokenization_spaces is not None:
					lowerCamelCase					:      Dict          =   clean_up_tokenization_spaces

			if stop_sequence is not None:
					lowerCamelCase					:      int          =   self.tokenizer.encode(UpperCAmelCase_		,			add_special_tokens=UpperCAmelCase_ )
					if len(UpperCAmelCase_ ) > 1:
							warnings.warn(
							    'Stopping on a multiple token sequence is not yet supported on transformers. The first token of'
							    ' the stop sequence will be used as the stop sequence string in the interim.' )
					lowerCamelCase					:      str          =   stop_sequence_ids[0]

			return preprocess_params, forward_params, postprocess_params




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_ )		->							List[str]:
			return True




	def       _UpperCamelCase   (			self		,			*UpperCAmelCase_		,			UpperCAmelCase_ )		->							Optional[Any]:
			lowerCamelCase					:      Optional[int]          =   self.model.config.prefix if self.model.config.prefix is not None else ''
			if isinstance(args[0]		,			UpperCAmelCase_ ):
					if self.tokenizer.pad_token_id is None:
							raise ValueError('Please make sure that the tokenizer has a pad_token_id when using a batch input' )
					lowerCamelCase					:      Any          =   ([prefix + arg for arg in args[0]],)
					lowerCamelCase					:      Tuple          =   True

			elif isinstance(args[0]		,			UpperCAmelCase_ ):
					lowerCamelCase					:      Optional[int]          =   (prefix + args[0],)
					lowerCamelCase					:      Tuple          =   False
			else:
					raise ValueError(
					    F""" `args[0]`: {args[0]} have the wrong format. The should be either of type `str` or type `list`""" )
			lowerCamelCase					:      Optional[Any]          =   self.tokenizer(*UpperCAmelCase_		,			padding=UpperCAmelCase_		,			truncation=UpperCAmelCase_		,			return_tensors=self.framework )
			# This is produced by tokenizers but is an invalid generate kwargs
			if "token_type_ids" in inputs:
					del inputs["token_type_ids"]
			return inputs




	def __call__(			self		,			*UpperCAmelCase_		,			**UpperCAmelCase_ )		->							Optional[Any]:
			lowerCamelCase					:      List[Any]          =   super().__call__(*UpperCAmelCase_		,			**UpperCAmelCase_ )
			if (
			    isinstance(args[0]		,			UpperCAmelCase_ )
			    and all(isinstance(UpperCAmelCase_		,			UpperCAmelCase_ ) for el in args[0] )
			    and all(len(UpperCAmelCase_ ) == 1 for res in result )
			):
					return [res[0] for res in result]
			return result




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_=TruncationStrategy.DO_NOT_TRUNCATE		,			**UpperCAmelCase_ )		->							Dict:
			lowerCamelCase					:      List[Any]          =   self._parse_and_tokenize(UpperCAmelCase_		,			truncation=UpperCAmelCase_		,			**UpperCAmelCase_ )
			return inputs




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			**UpperCAmelCase_ )		->							Union[str, Any]:
			if self.framework == "pt":
					lowerCamelCase					:      Tuple          =   model_inputs['input_ids'].shape
			elif self.framework == "tf":
					lowerCamelCase					:      Any          =   tf.shape(model_inputs['input_ids'] ).numpy()

			lowerCamelCase					:      List[Any]          =   generate_kwargs.get('min_length'		,			self.model.config.min_length )
			lowerCamelCase					:      Dict          =   generate_kwargs.get('max_length'		,			self.model.config.max_length )
			self.check_inputs(UpperCAmelCase_		,			generate_kwargs['min_length']		,			generate_kwargs['max_length'] )
			lowerCamelCase					:      Any          =   self.model.generate(**UpperCAmelCase_		,			**UpperCAmelCase_ )
			lowerCamelCase					:      str          =   output_ids.shape[0]
			if self.framework == "pt":
					lowerCamelCase					:      Tuple          =   output_ids.reshape(UpperCAmelCase_		,			out_b // in_b		,			*output_ids.shape[1:] )
			elif self.framework == "tf":
					lowerCamelCase					:      Tuple          =   tf.reshape(UpperCAmelCase_		,			(in_b, out_b // in_b, *output_ids.shape[1:]) )
			return {"output_ids": output_ids}




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_=ReturnType.TEXT		,			UpperCAmelCase_=False )		->							Tuple:
			lowerCamelCase					:      Union[str, Any]          =   []
			for output_ids in model_outputs["output_ids"][0]:
					if return_type == ReturnType.TENSORS:
							lowerCamelCase					:      Any          =   {F"""{self.return_name}_token_ids""": output_ids}
					elif return_type == ReturnType.TEXT:
							lowerCamelCase					:      str          =   {
							    F"""{self.return_name}_text""": self.tokenizer.decode(
							        UpperCAmelCase_		,			skip_special_tokens=UpperCAmelCase_		,			clean_up_tokenization_spaces=UpperCAmelCase_		,			)
							}
					records.append(UpperCAmelCase_ )
			return records






@add_end_docstrings(__UpperCAmelCase		)
class 	_lowercase       (    __UpperCAmelCase		):
	lowercase_				=     'summary'




	def __call__(			self		,			*UpperCAmelCase_		,			**UpperCAmelCase_ )		->							Any:
			return super().__call__(*UpperCAmelCase_		,			**UpperCAmelCase_ )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_ )		->							bool:
			if max_length < min_length:
					logger.warning(F"""Your min_length={min_length} must be inferior than your max_length={max_length}.""" )

			if input_length < max_length:
					logger.warning(
					    F"""Your max_length is set to {max_length}, but your input_length is only {input_length}. Since this is """
					    'a summarization task, where outputs shorter than the input are typically wanted, you might '
					    F"""consider decreasing max_length manually, e.g. summarizer('...', max_length={input_length//2})""" )






@add_end_docstrings(__UpperCAmelCase		)
class 	_lowercase       (    __UpperCAmelCase		):
	lowercase_				=     'translation'




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_ )		->							str:
			if input_length > 0.9 * max_length:
					logger.warning(
					    F"""Your input_length: {input_length} is bigger than 0.9 * max_length: {max_length}. You might consider """
					    'increasing your max_length manually, e.g. translator(\'...\', max_length=400)' )
			return True




	def       _UpperCamelCase   (			self		,			*UpperCAmelCase_		,			UpperCAmelCase_=TruncationStrategy.DO_NOT_TRUNCATE		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None )		->							List[Any]:
			if getattr(self.tokenizer		,			'_build_translation_inputs'		,			UpperCAmelCase_ ):
					return self.tokenizer._build_translation_inputs(
					    *UpperCAmelCase_		,			return_tensors=self.framework		,			truncation=UpperCAmelCase_		,			src_lang=UpperCAmelCase_		,			tgt_lang=UpperCAmelCase_ )
			else:
					return super()._parse_and_tokenize(*UpperCAmelCase_		,			truncation=UpperCAmelCase_ )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			**UpperCAmelCase_ )		->							List[Any]:
			lowerCamelCase					:      Dict          =   super()._sanitize_parameters(**UpperCAmelCase_ )
			if src_lang is not None:
					lowerCamelCase					:      Dict          =   src_lang
			if tgt_lang is not None:
					lowerCamelCase					:      List[str]          =   tgt_lang
			if src_lang is None and tgt_lang is None:
					# Backward compatibility, direct arguments use is preferred.
					lowerCamelCase					:      Tuple          =   kwargs.get('task'		,			self.task )
					lowerCamelCase					:      str          =   task.split('_' )
					if task and len(UpperCAmelCase_ ) == 4:
							# translation, XX, to YY
							lowerCamelCase					:      List[Any]          =   items[1]
							lowerCamelCase					:      str          =   items[3]
			return preprocess_params, forward_params, postprocess_params




	def __call__(			self		,			*UpperCAmelCase_		,			**UpperCAmelCase_ )		->							Union[str, Any]:
			return super().__call__(*UpperCAmelCase_		,			**UpperCAmelCase_ )








"""simple docstring"""

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


_A						=			logging.get_logger(__name__)


_A						=			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all BART models at https://huggingface.co/models?filter=bart
_A						=			{
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json',
    },
}

_A						=			{
    'facebook/bart-base': 1_0_2_4,
    'facebook/bart-large': 1_0_2_4,
    'facebook/bart-large-mnli': 1_0_2_4,
    'facebook/bart-large-cnn': 1_0_2_4,
    'facebook/bart-large-xsum': 1_0_2_4,
    'yjernite/bart_eli5': 1_0_2_4,
}






class 	_lowercase       (    __UpperCAmelCase		):
	lowercase_				=     VOCAB_FILES_NAMES
	lowercase_				=     PRETRAINED_VOCAB_FILES_MAP
	lowercase_				=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase_				=     ['input_ids', 'attention_mask']
	lowercase_				=     BartTokenizer




	def __init__(			self		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			UpperCAmelCase_="replace"		,			UpperCAmelCase_="<s>"		,			UpperCAmelCase_="</s>"		,			UpperCAmelCase_="</s>"		,			UpperCAmelCase_="<s>"		,			UpperCAmelCase_="<unk>"		,			UpperCAmelCase_="<pad>"		,			UpperCAmelCase_="<mask>"		,			UpperCAmelCase_=False		,			UpperCAmelCase_=True		,			**UpperCAmelCase_		,			)		->							Union[str, Any]:
			super().__init__(
			    UpperCAmelCase_		,			UpperCAmelCase_		,			tokenizer_file=UpperCAmelCase_		,			errors=UpperCAmelCase_		,			bos_token=UpperCAmelCase_		,			eos_token=UpperCAmelCase_		,			sep_token=UpperCAmelCase_		,			cls_token=UpperCAmelCase_		,			unk_token=UpperCAmelCase_		,			pad_token=UpperCAmelCase_		,			mask_token=UpperCAmelCase_		,			add_prefix_space=UpperCAmelCase_		,			trim_offsets=UpperCAmelCase_		,			**UpperCAmelCase_		,			)

			lowerCamelCase					:      List[Any]          =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
			if pre_tok_state.get('add_prefix_space'		,			UpperCAmelCase_ ) != add_prefix_space:
					lowerCamelCase					:      Tuple          =   getattr(UpperCAmelCase_		,			pre_tok_state.pop('type' ) )
					lowerCamelCase					:      Optional[Any]          =   add_prefix_space
					lowerCamelCase					:      str          =   pre_tok_class(**UpperCAmelCase_ )

			lowerCamelCase					:      Optional[Any]          =   add_prefix_space

			# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
			lowerCamelCase					:      Dict          =   'post_processor'
			lowerCamelCase					:      str          =   getattr(self.backend_tokenizer		,			UpperCAmelCase_		,			UpperCAmelCase_ )
			if tokenizer_component_instance:
					lowerCamelCase					:      Optional[Any]          =   json.loads(tokenizer_component_instance.__getstate__() )

					# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
					if "sep" in state:
							lowerCamelCase					:      int          =   tuple(state['sep'] )
					if "cls" in state:
							lowerCamelCase					:      str          =   tuple(state['cls'] )

					lowerCamelCase					:      Optional[Any]          =   False

					if state.get('add_prefix_space'		,			UpperCAmelCase_ ) != add_prefix_space:
							lowerCamelCase					:      Dict          =   add_prefix_space
							lowerCamelCase					:      Tuple          =   True

					if state.get('trim_offsets'		,			UpperCAmelCase_ ) != trim_offsets:
							lowerCamelCase					:      Tuple          =   trim_offsets
							lowerCamelCase					:      Dict          =   True

					if changes_to_apply:
							lowerCamelCase					:      Optional[int]          =   getattr(UpperCAmelCase_		,			state.pop('type' ) )
							lowerCamelCase					:      Any          =   component_class(**UpperCAmelCase_ )
							setattr(self.backend_tokenizer		,			UpperCAmelCase_		,			UpperCAmelCase_ )




	@property
	def       _UpperCamelCase   (			self )		->							str:
			if self._mask_token is None:
					if self.verbose:
							logger.error('Using mask_token, but it is not set yet.' )
					return None
			return str(self._mask_token )




	@mask_token.setter
	def       _UpperCamelCase   (			self		,			UpperCAmelCase_ )		->							List[Any]:
			lowerCamelCase					:      Optional[int]          =   AddedToken(UpperCAmelCase_		,			lstrip=UpperCAmelCase_		,			rstrip=UpperCAmelCase_ ) if isinstance(UpperCAmelCase_		,			UpperCAmelCase_ ) else value
			lowerCamelCase					:      int          =   value




	def       _UpperCamelCase   (			self		,			*UpperCAmelCase_		,			**UpperCAmelCase_ )		->							BatchEncoding:
			lowerCamelCase					:      str          =   kwargs.get('is_split_into_words'		,			UpperCAmelCase_ )

			if is_split_into_words and not self.add_prefix_space:
					raise ValueError(
					    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
					    'to use it with pretokenized inputs.' )

			return super()._batch_encode_plus(*UpperCAmelCase_		,			**UpperCAmelCase_ )




	def       _UpperCamelCase   (			self		,			*UpperCAmelCase_		,			**UpperCAmelCase_ )		->							BatchEncoding:
			lowerCamelCase					:      Optional[Any]          =   kwargs.get('is_split_into_words'		,			UpperCAmelCase_ )

			if is_split_into_words and not self.add_prefix_space:
					raise ValueError(
					    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
					    'to use it with pretokenized inputs.' )

			return super()._encode_plus(*UpperCAmelCase_		,			**UpperCAmelCase_ )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ = None )		->							Tuple[str]:
			lowerCamelCase					:      Any          =   self._tokenizer.model.save(UpperCAmelCase_		,			name=UpperCAmelCase_ )
			return tuple(UpperCAmelCase_ )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_=None )		->							List[Any]:
			lowerCamelCase					:      Union[str, Any]          =   [self.bos_token_id] + token_ids_a + [self.eos_token_id]
			if token_ids_a is None:
					return output

			return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ = None )		->							List[int]:
			lowerCamelCase					:      List[Any]          =   [self.sep_token_id]
			lowerCamelCase					:      str          =   [self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep ) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]






