





from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
    from ...feature_extraction_utils import FeatureExtractionMixin
    from ...tokenization_utils_base import PreTrainedTokenizerBase
    from ...utils import TensorType

snake_case_           =      logging.get_logger(__name__)

snake_case_           =      {
    'openai/whisper-base': 'https://huggingface.co/openai/whisper-base/resolve/main/config.json',
}

# fmt: off
snake_case_           =      [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786,
    11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791,
    17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409,
    34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361
]
snake_case_           =      [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793,
    14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675,
    22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865,
    42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362
]

class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):
      A_						:    Optional[int]							=   'whisper'
      A_						:    Optional[Any]							=   ['past_key_values']
      A_						:    Any							=   {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}


      def __init__(self	:				List[str]    ,      a__	:				Optional[Any]=5_1865    ,      a__	:				str=80    ,      a__	:				List[Any]=6    ,      a__	:				Dict=4    ,      a__	:				Union[str, Any]=6    ,      a__	:				Dict=4    ,      a__	:				Optional[int]=1536    ,      a__	:				Union[str, Any]=1536    ,      a__	:				Optional[int]=0.0    ,      a__	:				Tuple=0.0    ,      a__	:				Dict=5_0257    ,      a__	:				List[Any]=True    ,      a__	:				List[str]=True    ,      a__	:				Optional[int]="gelu"    ,      a__	:				Optional[int]=256    ,      a__	:				Union[str, Any]=0.0    ,      a__	:				int=0.0    ,      a__	:				str=0.0    ,      a__	:				Tuple=0.0_2    ,      a__	:				List[Any]=False    ,      a__	:				Tuple=1500    ,      a__	:				List[str]=448    ,      a__	:				List[Any]=5_0256    ,      a__	:				List[Any]=5_0256    ,      a__	:				Dict=5_0256    ,      a__	:				Optional[int]=None    ,      a__	:				Tuple=[220, 5_0256]    ,      a__	:				str=False    ,      a__	:				str=256    ,      a__	:				Dict=False    ,      a__	:				Tuple=0.0_5    ,      a__	:				List[str]=10    ,      a__	:				Optional[Any]=2    ,      a__	:				List[Any]=0.0    ,      a__	:				Optional[int]=10    ,      a__	:				Optional[Any]=0    ,      a__	:				Tuple=7    ,      **a__	:				int    ,      ):





         """simple docstring"""



         __snake_case								=       vocab_size
         __snake_case								=       num_mel_bins
         __snake_case								=       d_model
         __snake_case								=       encoder_layers
         __snake_case								=       encoder_attention_heads
         __snake_case								=       decoder_layers
         __snake_case								=       decoder_attention_heads
         __snake_case								=       decoder_ffn_dim
         __snake_case								=       encoder_ffn_dim
         __snake_case								=       dropout
         __snake_case								=       attention_dropout
         __snake_case								=       activation_dropout
         __snake_case								=       activation_function
         __snake_case								=       init_std
         __snake_case								=       encoder_layerdrop
         __snake_case								=       decoder_layerdrop
         __snake_case								=       use_cache
         __snake_case								=       encoder_layers
         __snake_case								=       scale_embedding  # scale factor will be sqrt(d_model) if True
         __snake_case								=       max_source_positions
         __snake_case								=       max_target_positions

         # Audio Classification-specific parameters. Feel free to ignore for other classes.
         __snake_case								=       classifier_proj_size
         __snake_case								=       use_weighted_layer_sum

         # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
         __snake_case								=       apply_spec_augment
         __snake_case								=       mask_time_prob
         __snake_case								=       mask_time_length
         __snake_case								=       mask_time_min_masks
         __snake_case								=       mask_feature_prob
         __snake_case								=       mask_feature_length
         __snake_case								=       mask_feature_min_masks

         __snake_case								=       median_filter_width

         super().__init__(
             pad_token_id=a__    ,      bos_token_id=a__    ,      eos_token_id=a__    ,      is_encoder_decoder=a__    ,      decoder_start_token_id=a__    ,      suppress_tokens=a__    ,      begin_suppress_tokens=a__    ,      **a__    ,      )






class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):


      @property
      def        a			(self	:				str				):





         """simple docstring"""



         __snake_case								=       OrderedDict(
             [
                 ('''input_features''', {0: '''batch''', 1: '''feature_size''', 2: '''encoder_sequence'''}),
             ]				)
         if self.use_past:
            __snake_case								=       {0: '''batch'''}
         else:
            __snake_case								=       {0: '''batch''', 1: '''decoder_sequence'''}

         if self.use_past:
            self.fill_with_past_key_values_(a__    ,      direction='''inputs'''				)

         return common_inputs


      def        a			(self	:				List[Any]    ,      a__	:				Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]    ,      a__	:				int = -1    ,      a__	:				int = -1    ,      a__	:				bool = False    ,      a__	:				Optional["TensorType"] = None    ,      a__	:				int = 2_2050    ,      a__	:				float = 5.0    ,      a__	:				int = 220    ,      ):





         """simple docstring"""



         __snake_case								=       OrderedDict()
         __snake_case								=       OnnxConfig.generate_dummy_inputs(
             self    ,      preprocessor=preprocessor.feature_extractor    ,      batch_size=a__    ,      framework=a__    ,      sampling_rate=a__    ,      time_duration=a__    ,      frequency=a__    ,      )
         __snake_case								=       encoder_inputs['''input_features'''].shape[2]
         __snake_case								=       encoder_sequence_length // 2 if self.use_past else seq_length

         __snake_case								=       super().generate_dummy_inputs(
             preprocessor.tokenizer    ,      a__    ,      a__    ,      a__    ,      a__				)

         __snake_case								=       encoder_inputs.pop('''input_features'''				)
         __snake_case								=       decoder_inputs.pop('''decoder_input_ids'''				)

         if "past_key_values" in decoder_inputs:
            __snake_case								=       decoder_inputs.pop('''past_key_values'''				)

         return dummy_inputs


      @property
      def        a			(self	:				Optional[Any]				):





         """simple docstring"""



         return 1E-3








import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin

class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase						,				unittest.TestCase  ):
      A_						:    int							=   XLMTokenizer
      A_						:    Optional[Any]							=   False


      def        a			(self	:				Optional[int]				):





         """simple docstring"""



         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         __snake_case								=       [
             '''l''',
             '''o''',
             '''w''',
             '''e''',
             '''r''',
             '''s''',
             '''t''',
             '''i''',
             '''d''',
             '''n''',
             '''w</w>''',
             '''r</w>''',
             '''t</w>''',
             '''lo''',
             '''low''',
             '''er</w>''',
             '''low</w>''',
             '''lowest</w>''',
             '''newer</w>''',
             '''wider</w>''',
             '''<unk>''',
         ]
         __snake_case								=       dict(zip(a__    ,      range(len(a__				)				)				)				)
         __snake_case								=       ['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

         __snake_case								=       os.path.join(self.tmpdirname    ,      VOCAB_FILES_NAMES['''vocab_file''']				)
         __snake_case								=       os.path.join(self.tmpdirname    ,      VOCAB_FILES_NAMES['''merges_file''']				)
         with open(self.vocab_file    ,      '''w'''				) as fp:
            fp.write(json.dumps(a__				)				)
         with open(self.merges_file    ,      '''w'''				) as fp:
            fp.write('''\n'''.join(a__				)				)


      def        a			(self	:				int    ,      a__	:				Any				):





         """simple docstring"""



         __snake_case								=       '''lower newer'''
         __snake_case								=       '''lower newer'''
         return input_text, output_text


      def        a			(self	:				Optional[Any]				):





         """simple docstring"""



         __snake_case								=       XLMTokenizer(self.vocab_file    ,      self.merges_file				)

         __snake_case								=       '''lower'''
         __snake_case								=       ['''low''', '''er</w>''']
         __snake_case								=       tokenizer.tokenize(a__				)
         self.assertListEqual(a__    ,      a__				)

         __snake_case								=       tokens + ['''<unk>''']
         __snake_case								=       [14, 15, 20]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a__				)    ,      a__				)


      @slow
      def        a			(self	:				Union[str, Any]				):





         """simple docstring"""



         __snake_case								=       XLMTokenizer.from_pretrained('''xlm-mlm-en-2048'''				)

         __snake_case								=       tokenizer.encode('''sequence builders'''    ,      add_special_tokens=a__				)
         __snake_case								=       tokenizer.encode('''multi-sequence build'''    ,      add_special_tokens=a__				)

         __snake_case								=       tokenizer.build_inputs_with_special_tokens(a__				)
         __snake_case								=       tokenizer.build_inputs_with_special_tokens(a__    ,      a__				)

         assert encoded_sentence == [0] + text + [1]
         assert encoded_pair == [0] + text + [1] + text_a + [1]

