





'''simple docstring'''



import enum
import warnings

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
	import tensorflow as tf

	from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

if is_torch_available():
	from ..models.auto.modeling_auto import MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)






class 							__lowercase    ( enum.Enum  ):







	'''simple docstring'''

	a      :			str     =    0
	a      :			Dict     =    1






@add_end_docstrings(__UpperCamelCase  )
class 							__lowercase    ( __UpperCamelCase  ):







	'''simple docstring'''

	a      :			List[str]     =    "generated"


	def __init__(self       ,*_lowerCamelCase       ,**_lowerCamelCase	)							->							Optional[Any]:







				'''simple docstring'''





				super().__init__(*_lowerCAmelCase       ,**_lowerCAmelCase	)

				self.check_model_type(
				    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
				    if self.framework == '''tf'''
				    else MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase=None       ,_lowerCamelCase=None       ,_lowerCamelCase=None       ,_lowerCamelCase=None       ,_lowerCamelCase=None       ,_lowerCamelCase=None       ,**_lowerCamelCase       ,)							->							Optional[Any]:







				'''simple docstring'''





				__lowercase			  =							{}
				if truncation is not None:
							__lowercase			  =							truncation

				__lowercase			  =							generate_kwargs

				__lowercase			  =							{}
				if return_tensors is not None and return_type is None:
							__lowercase			  =							ReturnType.TENSORS if return_tensors else ReturnType.TEXT
				if return_type is not None:
							__lowercase			  =							return_type

				if clean_up_tokenization_spaces is not None:
							__lowercase			  =							clean_up_tokenization_spaces

				if stop_sequence is not None:
							__lowercase			  =							self.tokenizer.encode(_lowerCAmelCase       ,add_special_tokens=_lowerCAmelCase	)
							if len(_lowerCAmelCase	) > 1:
										warnings.warn(
										    '''Stopping on a multiple token sequence is not yet supported on transformers. The first token of'''
										    ''' the stop sequence will be used as the stop sequence string in the interim.'''	)
							__lowercase			  =							stop_sequence_ids[0]

				return preprocess_params, forward_params, postprocess_params


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase	)							->							Optional[int]:







				'''simple docstring'''





				return True


	def   _UpperCAmelCase       (self       ,*_lowerCamelCase       ,_lowerCamelCase	)							->							int:







				'''simple docstring'''





				__lowercase			  =							self.model.config.prefix if self.model.config.prefix is not None else """"""
				if isinstance(args[0]       ,_lowerCAmelCase	):
							if self.tokenizer.pad_token_id is None:
										raise ValueError('''Please make sure that the tokenizer has a pad_token_id when using a batch input'''	)
							__lowercase			  =							([prefix + arg for arg in args[0]],)
							__lowercase			  =							True

				elif isinstance(args[0]       ,_lowerCAmelCase	):
							__lowercase			  =							(prefix + args[0],)
							__lowercase			  =							False
				else:
							raise ValueError(
							    f" `args[0]`: {args[0]} have the wrong format. The should be either of type `str` or type `list`"	)
				__lowercase			  =							self.tokenizer(*_lowerCAmelCase       ,padding=_lowerCAmelCase       ,truncation=_lowerCAmelCase       ,return_tensors=self.framework	)
				# This is produced by tokenizers but is an invalid generate kwargs
				if "token_type_ids" in inputs:
							del inputs["token_type_ids"]
				return inputs


	def __call__(self       ,*_lowerCamelCase       ,**_lowerCamelCase	)							->							Optional[Any]:







				'''simple docstring'''





				__lowercase			  =							super().__call__(*_lowerCAmelCase       ,**_lowerCAmelCase	)
				if (
				    isinstance(args[0]       ,_lowerCAmelCase	)
				    and all(isinstance(_lowerCAmelCase       ,_lowerCAmelCase	) for el in args[0]	)
				    and all(len(_lowerCAmelCase	) == 1 for res in result	)
				):
							return [res[0] for res in result]
				return result


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase=TruncationStrategy.DO_NOT_TRUNCATE       ,**_lowerCamelCase	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							self._parse_and_tokenize(_lowerCAmelCase       ,truncation=_lowerCAmelCase       ,**_lowerCAmelCase	)
				return inputs


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,**_lowerCamelCase	)							->							Optional[int]:







				'''simple docstring'''





				if self.framework == "pt":
							__lowercase			  =							model_inputs["""input_ids"""].shape
				elif self.framework == "tf":
							__lowercase			  =							tf.shape(model_inputs['''input_ids''']	).numpy()

				__lowercase			  =							generate_kwargs.get('''min_length'''       ,self.model.config.min_length	)
				__lowercase			  =							generate_kwargs.get('''max_length'''       ,self.model.config.max_length	)
				self.check_inputs(_lowerCAmelCase       ,generate_kwargs['''min_length''']       ,generate_kwargs['''max_length''']	)
				__lowercase			  =							self.model.generate(**_lowerCAmelCase       ,**_lowerCAmelCase	)
				__lowercase			  =							output_ids.shape[0]
				if self.framework == "pt":
							__lowercase			  =							output_ids.reshape(_lowerCAmelCase       ,out_b // in_b       ,*output_ids.shape[1:]	)
				elif self.framework == "tf":
							__lowercase			  =							tf.reshape(_lowerCAmelCase       ,(in_b, out_b // in_b, *output_ids.shape[1:])	)
				return {"output_ids": output_ids}


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase=ReturnType.TEXT       ,_lowerCamelCase=False	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							[]
				for output_ids in model_outputs["output_ids"][0]:
							if return_type == ReturnType.TENSORS:
										__lowercase			  =							{f"{self.return_name}_token_ids": output_ids}
							elif return_type == ReturnType.TEXT:
										__lowercase			  =							{
										    f"{self.return_name}_text": self.tokenizer.decode(
										        _lowerCAmelCase       ,skip_special_tokens=_lowerCAmelCase       ,clean_up_tokenization_spaces=_lowerCAmelCase       ,)
										}
							records.append(_lowerCAmelCase	)
				return records






@add_end_docstrings(__UpperCamelCase  )
class 							__lowercase    ( __UpperCamelCase  ):







	'''simple docstring'''

	a      :			int     =    "summary"


	def __call__(self       ,*_lowerCamelCase       ,**_lowerCamelCase	)							->							Dict:







				'''simple docstring'''





				return super().__call__(*_lowerCAmelCase       ,**_lowerCAmelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase	)							->							Union[str, Any]:







				'''simple docstring'''





				if max_length < min_length:
							logger.warning(f"Your min_length={min_length} must be inferior than your max_length={max_length}."	)

				if input_length < max_length:
							logger.warning(
							    f"Your max_length is set to {max_length}, but your input_length is only {input_length}. Since this is "
							    '''a summarization task, where outputs shorter than the input are typically wanted, you might '''
							    f"consider decreasing max_length manually, e.g. summarizer(\'...\', max_length={input_length//2})"	)






@add_end_docstrings(__UpperCamelCase  )
class 							__lowercase    ( __UpperCamelCase  ):







	'''simple docstring'''

	a      :			Tuple     =    "translation"


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase       ,_lowerCamelCase	)							->							Any:







				'''simple docstring'''





				if input_length > 0.9 * max_length:
							logger.warning(
							    f"Your input_length: {input_length} is bigger than 0.9 * max_length: {max_length}. You might consider "
							    '''increasing your max_length manually, e.g. translator(\'...\', max_length=400)'''	)
				return True


	def   _UpperCAmelCase       (self       ,*_lowerCamelCase       ,_lowerCamelCase=TruncationStrategy.DO_NOT_TRUNCATE       ,_lowerCamelCase=None       ,_lowerCamelCase=None	)							->							Union[str, Any]:







				'''simple docstring'''





				if getattr(self.tokenizer       ,'''_build_translation_inputs'''       ,_lowerCAmelCase	):
							return self.tokenizer._build_translation_inputs(
							    *_lowerCAmelCase       ,return_tensors=self.framework       ,truncation=_lowerCAmelCase       ,src_lang=_lowerCAmelCase       ,tgt_lang=_lowerCAmelCase	)
				else:
							return super()._parse_and_tokenize(*_lowerCAmelCase       ,truncation=_lowerCAmelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase=None       ,_lowerCamelCase=None       ,**_lowerCamelCase	)							->							str:







				'''simple docstring'''





				__lowercase			  =							super()._sanitize_parameters(**_lowerCAmelCase	)
				if src_lang is not None:
							__lowercase			  =							src_lang
				if tgt_lang is not None:
							__lowercase			  =							tgt_lang
				if src_lang is None and tgt_lang is None:
							# Backward compatibility, direct arguments use is preferred.
							__lowercase			  =							kwargs.get('''task'''       ,self.task	)
							__lowercase			  =							task.split('''_'''	)
							if task and len(_lowerCAmelCase	) == 4:
										# translation, XX, to YY
										__lowercase			  =							items[1]
										__lowercase			  =							items[3]
				return preprocess_params, forward_params, postprocess_params


	def __call__(self       ,*_lowerCamelCase       ,**_lowerCamelCase	)							->							List[Any]:







				'''simple docstring'''





				return super().__call__(*_lowerCAmelCase       ,**_lowerCAmelCase	)












'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =      {
    '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json''',
}






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	a      :			Union[str, Any]     =    "gpt_neox_japanese"


	def __init__(self       ,_lowerCamelCase=32000       ,_lowerCamelCase=2560       ,_lowerCamelCase=32       ,_lowerCamelCase=32       ,_lowerCamelCase=4       ,_lowerCamelCase="gelu"       ,_lowerCamelCase=1.0_0       ,_lowerCamelCase=10000       ,_lowerCamelCase=2048       ,_lowerCamelCase=0.0_2       ,_lowerCamelCase=1E-5       ,_lowerCamelCase=True       ,_lowerCamelCase=31996       ,_lowerCamelCase=31999       ,_lowerCamelCase=0.1       ,_lowerCamelCase=0.0       ,**_lowerCamelCase       ,)							->							Optional[int]:







				'''simple docstring'''





				super().__init__(bos_token_id=_lowerCamelCase       ,eos_token_id=_lowerCamelCase       ,**_lowerCamelCase	)
				__lowercase			  =							vocab_size
				__lowercase			  =							max_position_embeddings
				__lowercase			  =							hidden_size
				__lowercase			  =							num_hidden_layers
				__lowercase			  =							num_attention_heads
				__lowercase			  =							intermediate_multiple_size
				__lowercase			  =							hidden_act
				__lowercase			  =							rotary_pct
				__lowercase			  =							rotary_emb_base
				__lowercase			  =							initializer_range
				__lowercase			  =							layer_norm_eps
				__lowercase			  =							use_cache
				__lowercase			  =							attention_dropout
				__lowercase			  =							hidden_dropout





