'''simple docstring'''




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


__UpperCAmelCase												=logging.get_logger(__name__)

__UpperCAmelCase												={"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase												={
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

__UpperCAmelCase												={
    "gpt2": 1_0_2_4,
    "gpt2-medium": 1_0_2_4,
    "gpt2-large": 1_0_2_4,
    "gpt2-xl": 1_0_2_4,
    "distilgpt2": 1_0_2_4,
}




class    a__				(       UpperCAmelCase__			):
     lowerCamelCase  :					Any							=VOCAB_FILES_NAMES
     lowerCamelCase  :					Optional[Any]							=PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase  :					int							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase  :					Union[str, Any]							=["input_ids", "attention_mask"]
     lowerCamelCase  :					List[Any]							=GPTaTokenizer






     def __init__(			self		:		List[str]  ,    a		:		Dict=None  ,    a		:		Optional[Any]=None  ,    a		:		int=None  ,    a		:		Optional[int]="<|endoftext|>"  ,    a		:		List[str]="<|endoftext|>"  ,    a		:		Union[str, Any]="<|endoftext|>"  ,    a		:		Tuple=False  ,    **a		:		Any  ,    ):





         """simple docstring"""



         super().__init__(
             a  ,    a  ,    tokenizer_file=a  ,    unk_token=a  ,    bos_token=a  ,    eos_token=a  ,    add_prefix_space=a  ,    **a  ,    )

         __lowerCamelCase          =  kwargs.pop('''add_bos_token'''  ,    a   )

         __lowerCamelCase          =  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
         if pre_tok_state.get('''add_prefix_space'''  ,    a   ) != add_prefix_space:
             __lowerCamelCase          =  getattr(a  ,    pre_tok_state.pop('''type'''   )   )
             __lowerCamelCase          =  add_prefix_space
             __lowerCamelCase          =  pre_tok_class(**a   )

         __lowerCamelCase          =  add_prefix_space






     def        SCREAMING_SNAKE_CASE__       (			self		:		str  ,    *a		:		Optional[Any]  ,    **a		:		str   ):





         """simple docstring"""



         __lowerCamelCase          =  kwargs.get('''is_split_into_words'''  ,    a   )
         assert self.add_prefix_space or not is_split_into_words, (
             f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
             "to use it with pretokenized inputs."
         )

         return super()._batch_encode_plus(*a  ,    **a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int  ,    *a		:		List[str]  ,    **a		:		int   ):





         """simple docstring"""



         __lowerCamelCase          =  kwargs.get('''is_split_into_words'''  ,    a   )

         assert self.add_prefix_space or not is_split_into_words, (
             f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
             "to use it with pretokenized inputs."
         )

         return super()._encode_plus(*a  ,    **a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]  ,    a		:		str  ,    a		:		Optional[str] = None   ):





         """simple docstring"""



         __lowerCamelCase          =  self._tokenizer.model.save(a  ,    name=a   )
         return tuple(a   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		Optional[Any]  ,    a		:		"Conversation"   ):





         """simple docstring"""



         __lowerCamelCase          =  []
         for is_user, text in conversation.iter_texts():
             input_ids.extend(self.encode(a  ,    add_special_tokens=a   ) + [self.eos_token_id]   )

         if len(a   ) > self.model_max_length:
             __lowerCamelCase          =  input_ids[-self.model_max_length :]
         return input_ids

'''simple docstring'''




import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase												=logging.get_logger(__name__)

__UpperCAmelCase												={
    "microsoft/unispeech-sat-base-100h-libri-ft": (
        "https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json"
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}




class    a__				(       UpperCAmelCase__			):
     lowerCamelCase  :					List[Any]							="unispeech-sat"






     def __init__(			self		:		Dict  ,    a		:		str=32  ,    a		:		Any=7_68  ,    a		:		Optional[Any]=12  ,    a		:		Optional[int]=12  ,    a		:		int=30_72  ,    a		:		int="gelu"  ,    a		:		Dict=0.1  ,    a		:		Dict=0.1  ,    a		:		List[Any]=0.1  ,    a		:		Tuple=0.0  ,    a		:		Optional[Any]=0.0  ,    a		:		Tuple=0.1  ,    a		:		List[Any]=0.1  ,    a		:		str=0.02  ,    a		:		List[Any]=1e-5  ,    a		:		int="group"  ,    a		:		Union[str, Any]="gelu"  ,    a		:		Optional[int]=(5_12, 5_12, 5_12, 5_12, 5_12, 5_12, 5_12)  ,    a		:		List[Any]=(5, 2, 2, 2, 2, 2, 2)  ,    a		:		int=(10, 3, 3, 3, 3, 2, 2)  ,    a		:		Optional[Any]=False  ,    a		:		Any=1_28  ,    a		:		Tuple=16  ,    a		:		str=False  ,    a		:		Optional[Any]=True  ,    a		:		Dict=0.05  ,    a		:		List[Any]=10  ,    a		:		Any=2  ,    a		:		Optional[Any]=0.0  ,    a		:		Optional[Any]=10  ,    a		:		Any=0  ,    a		:		Any=3_20  ,    a		:		str=2  ,    a		:		List[str]=0.1  ,    a		:		List[str]=1_00  ,    a		:		List[str]=2_56  ,    a		:		str=2_56  ,    a		:		Dict=0.1  ,    a		:		Optional[Any]="mean"  ,    a		:		str=False  ,    a		:		Tuple=False  ,    a		:		Optional[Any]=2_56  ,    a		:		int=(5_12, 5_12, 5_12, 5_12, 15_00)  ,    a		:		int=(5, 3, 3, 1, 1)  ,    a		:		Any=(1, 2, 3, 1, 1)  ,    a		:		Union[str, Any]=5_12  ,    a		:		Optional[int]=0  ,    a		:		Optional[int]=1  ,    a		:		Optional[int]=2  ,    a		:		int=5_04  ,    **a		:		Dict  ,    ):





         """simple docstring"""



         super().__init__(**a  ,    pad_token_id=a  ,    bos_token_id=a  ,    eos_token_id=a   )
         __lowerCamelCase          =  hidden_size
         __lowerCamelCase          =  feat_extract_norm
         __lowerCamelCase          =  feat_extract_activation
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  conv_bias
         __lowerCamelCase          =  num_conv_pos_embeddings
         __lowerCamelCase          =  num_conv_pos_embedding_groups
         __lowerCamelCase          =  len(self.conv_dim   )
         __lowerCamelCase          =  num_hidden_layers
         __lowerCamelCase          =  intermediate_size
         __lowerCamelCase          =  hidden_act
         __lowerCamelCase          =  num_attention_heads
         __lowerCamelCase          =  hidden_dropout
         __lowerCamelCase          =  attention_dropout
         __lowerCamelCase          =  activation_dropout
         __lowerCamelCase          =  feat_proj_dropout
         __lowerCamelCase          =  final_dropout
         __lowerCamelCase          =  layerdrop
         __lowerCamelCase          =  layer_norm_eps
         __lowerCamelCase          =  initializer_range
         __lowerCamelCase          =  vocab_size
         __lowerCamelCase          =  num_clusters
         __lowerCamelCase          =  do_stable_layer_norm
         __lowerCamelCase          =  use_weighted_layer_sum

         if (
             (len(self.conv_stride   ) != self.num_feat_extract_layers)
             or (len(self.conv_kernel   ) != self.num_feat_extract_layers)
             or (len(self.conv_dim   ) != self.num_feat_extract_layers)
         ):
             raise ValueError(
                 '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                 ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                 f""" {len(self.conv_dim   )}`, `len(config.conv_stride) = {len(self.conv_stride   )}`,"""
                 f""" `len(config.conv_kernel) = {len(self.conv_kernel   )}`."""   )

         # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
         __lowerCamelCase          =  apply_spec_augment
         __lowerCamelCase          =  mask_time_prob
         __lowerCamelCase          =  mask_time_length
         __lowerCamelCase          =  mask_time_min_masks
         __lowerCamelCase          =  mask_feature_prob
         __lowerCamelCase          =  mask_feature_length
         __lowerCamelCase          =  mask_feature_min_masks

         # parameters for pretraining with codevector quantized representations
         __lowerCamelCase          =  num_codevectors_per_group
         __lowerCamelCase          =  num_codevector_groups
         __lowerCamelCase          =  contrastive_logits_temperature
         __lowerCamelCase          =  feat_quantizer_dropout
         __lowerCamelCase          =  num_negatives
         __lowerCamelCase          =  codevector_dim
         __lowerCamelCase          =  proj_codevector_dim
         __lowerCamelCase          =  diversity_loss_weight

         # ctc loss
         __lowerCamelCase          =  ctc_loss_reduction
         __lowerCamelCase          =  ctc_zero_infinity

         # SequenceClassification-specific parameter. Feel free to ignore for other classes.
         __lowerCamelCase          =  classifier_proj_size

         # XVector-specific parameters. Feel free to ignore for other classes.
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  xvector_output_dim






     @property
     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         return functools.reduce(operator.mul  ,    self.conv_stride  ,    1   )
