




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast





class 	lowerCamelCase_							(				unittest.TestCase ):


 '''simple docstring'''







 def 				lowerCAmelCase_  (							self  :		Optional[int]						):
       # A mock response for an HTTP head request to emulate server down
       SCREAMING_SNAKE_CASE_     						=      mock.Mock()
       SCREAMING_SNAKE_CASE_     						=      500
       SCREAMING_SNAKE_CASE_     						=      {}
       SCREAMING_SNAKE_CASE_     						=      HTTPError
       SCREAMING_SNAKE_CASE_     						=      {}

       # Download this model to make sure it's in the cache.
       SCREAMING_SNAKE_CASE_     						=      BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'						)

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch('requests.Session.request'    ,    return_value=_lowerCamelCase						) as mock_head:
             SCREAMING_SNAKE_CASE_     						=      BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'						)
             # This check we did call the fake head request
             mock_head.assert_called()



 @require_tokenizers
 def 				lowerCAmelCase_  (							self  :		List[Any]						):
       # A mock response for an HTTP head request to emulate server down
       SCREAMING_SNAKE_CASE_     						=      mock.Mock()
       SCREAMING_SNAKE_CASE_     						=      500
       SCREAMING_SNAKE_CASE_     						=      {}
       SCREAMING_SNAKE_CASE_     						=      HTTPError
       SCREAMING_SNAKE_CASE_     						=      {}

       # Download this model to make sure it's in the cache.
       SCREAMING_SNAKE_CASE_     						=      GPTaTokenizerFast.from_pretrained('gpt2'						)

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch('requests.Session.request'    ,    return_value=_lowerCamelCase						) as mock_head:
             SCREAMING_SNAKE_CASE_     						=      GPTaTokenizerFast.from_pretrained('gpt2'						)
             # This check we did call the fake head request
             mock_head.assert_called()



 def 				lowerCAmelCase_  (							self  :		Union[str, Any]						):
       # This test is for deprecated behavior and can be removed in v5
       try:
             SCREAMING_SNAKE_CASE_     						=      tempfile.mktemp()
             with open(_lowerCamelCase    ,    'wb'						) as f:
                   http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'    ,    _lowerCamelCase						)

             SCREAMING_SNAKE_CASE_     						=      AlbertTokenizer.from_pretrained(_lowerCamelCase						)
       finally:
             os.remove(_lowerCamelCase						)

       # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
       # the current folder and have the right name.
       if os.path.isfile('tokenizer.json'						):
             # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
             return
       try:
             with open('tokenizer.json'    ,    'wb'						) as f:
                   http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'    ,    _lowerCamelCase						)
             SCREAMING_SNAKE_CASE_     						=      AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'						)
             # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
             self.assertEqual(tokenizer.vocab_size    ,    1_000						)
             # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

       finally:
             os.remove('tokenizer.json'						)




 def 				lowerCAmelCase_  (							self  :		int						):
       # This test is for deprecated behavior and can be removed in v5
       SCREAMING_SNAKE_CASE_     						=      AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'						)





@is_staging_test
class 	lowerCamelCase_							(				unittest.TestCase ):


 '''simple docstring'''







 lowercase_       =   ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']



 @classmethod
 def 				lowerCAmelCase_  (							cls  :		str						):
       SCREAMING_SNAKE_CASE_     						=      TOKEN
       HfFolder.save_token(_lowerCamelCase						)



 @classmethod
 def 				lowerCAmelCase_  (							cls  :		List[str]						):
       try:
             delete_repo(token=cls._token    ,    repo_id='test-tokenizer'						)
       except HTTPError:
             pass

       try:
             delete_repo(token=cls._token    ,    repo_id='valid_org/test-tokenizer-org'						)
       except HTTPError:
             pass

       try:
             delete_repo(token=cls._token    ,    repo_id='test-dynamic-tokenizer'						)
       except HTTPError:
             pass



 def 				lowerCAmelCase_  (							self  :		Union[str, Any]						):
       with tempfile.TemporaryDirectory() as tmp_dir:
             SCREAMING_SNAKE_CASE_     						=      os.path.join(_lowerCamelCase    ,    'vocab.txt'						)
             with open(_lowerCamelCase    ,    'w'    ,    encoding='utf-8'						) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
             SCREAMING_SNAKE_CASE_     						=      BertTokenizer(_lowerCamelCase						)

       tokenizer.push_to_hub('test-tokenizer'    ,    use_auth_token=self._token						)
       SCREAMING_SNAKE_CASE_     						=      BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"						)
       self.assertDictEqual(new_tokenizer.vocab    ,    tokenizer.vocab						)

       # Reset repo
       delete_repo(token=self._token    ,    repo_id='test-tokenizer'						)

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(_lowerCamelCase    ,    repo_id='test-tokenizer'    ,    push_to_hub=_lowerCamelCase    ,    use_auth_token=self._token						)

       SCREAMING_SNAKE_CASE_     						=      BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"						)
       self.assertDictEqual(new_tokenizer.vocab    ,    tokenizer.vocab						)



 def 				lowerCAmelCase_  (							self  :		str						):
       with tempfile.TemporaryDirectory() as tmp_dir:
             SCREAMING_SNAKE_CASE_     						=      os.path.join(_lowerCamelCase    ,    'vocab.txt'						)
             with open(_lowerCamelCase    ,    'w'    ,    encoding='utf-8'						) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
             SCREAMING_SNAKE_CASE_     						=      BertTokenizer(_lowerCamelCase						)

       tokenizer.push_to_hub('valid_org/test-tokenizer-org'    ,    use_auth_token=self._token						)
       SCREAMING_SNAKE_CASE_     						=      BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'						)
       self.assertDictEqual(new_tokenizer.vocab    ,    tokenizer.vocab						)

       # Reset repo
       delete_repo(token=self._token    ,    repo_id='valid_org/test-tokenizer-org'						)

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(
                 _lowerCamelCase    ,    repo_id='valid_org/test-tokenizer-org'    ,    push_to_hub=_lowerCamelCase    ,    use_auth_token=self._token						)

       SCREAMING_SNAKE_CASE_     						=      BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'						)
       self.assertDictEqual(new_tokenizer.vocab    ,    tokenizer.vocab						)




 @require_tokenizers
 def 				lowerCAmelCase_  (							self  :		Union[str, Any]						):
       CustomTokenizer.register_for_auto_class()
       with tempfile.TemporaryDirectory() as tmp_dir:
             SCREAMING_SNAKE_CASE_     						=      os.path.join(_lowerCamelCase    ,    'vocab.txt'						)
             with open(_lowerCamelCase    ,    'w'    ,    encoding='utf-8'						) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
             SCREAMING_SNAKE_CASE_     						=      CustomTokenizer(_lowerCamelCase						)

       # No fast custom tokenizer
       tokenizer.push_to_hub('test-dynamic-tokenizer'    ,    use_auth_token=self._token						)

       SCREAMING_SNAKE_CASE_     						=      AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"    ,    trust_remote_code=_lowerCamelCase						)
       # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__    ,    'CustomTokenizer'						)

       # Fast and slow custom tokenizer
       CustomTokenizerFast.register_for_auto_class()
       with tempfile.TemporaryDirectory() as tmp_dir:
             SCREAMING_SNAKE_CASE_     						=      os.path.join(_lowerCamelCase    ,    'vocab.txt'						)
             with open(_lowerCamelCase    ,    'w'    ,    encoding='utf-8'						) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)

             SCREAMING_SNAKE_CASE_     						=      BertTokenizerFast.from_pretrained(_lowerCamelCase						)
             bert_tokenizer.save_pretrained(_lowerCamelCase						)
             SCREAMING_SNAKE_CASE_     						=      CustomTokenizerFast.from_pretrained(_lowerCamelCase						)

       tokenizer.push_to_hub('test-dynamic-tokenizer'    ,    use_auth_token=self._token						)

       SCREAMING_SNAKE_CASE_     						=      AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"    ,    trust_remote_code=_lowerCamelCase						)
       # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__    ,    'CustomTokenizerFast'						)
       SCREAMING_SNAKE_CASE_     						=      AutoTokenizer.from_pretrained(
           F"{USER}/test-dynamic-tokenizer"    ,    use_fast=_lowerCamelCase    ,    trust_remote_code=_lowerCamelCase						)
       # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__    ,    'CustomTokenizer'						)





class 	lowerCamelCase_							(				unittest.TestCase ):


 '''simple docstring'''







 def 				lowerCAmelCase_  (							self  :		Any						):
       SCREAMING_SNAKE_CASE_     						=      Trie()
       trie.add('Hello 友達'						)
       self.assertEqual(trie.data    ,    {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}						)
       trie.add('Hello'						)
       trie.data
       self.assertEqual(trie.data    ,    {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}						)



 def 				lowerCAmelCase_  (							self  :		List[Any]						):
       SCREAMING_SNAKE_CASE_     						=      Trie()
       self.assertEqual(trie.split('[CLS] This is a extra_id_100'						)    ,    ['[CLS] This is a extra_id_100']						)
       trie.add('[CLS]'						)
       trie.add('extra_id_1'						)
       trie.add('extra_id_100'						)
       self.assertEqual(trie.split('[CLS] This is a extra_id_100'						)    ,    ['[CLS]', ' This is a ', 'extra_id_100']						)



 def 				lowerCAmelCase_  (							self  :		Dict						):
       SCREAMING_SNAKE_CASE_     						=      Trie()
       trie.add('A'						)
       self.assertEqual(trie.split('ABC'						)    ,    ['A', 'BC']						)
       self.assertEqual(trie.split('BCA'						)    ,    ['BC', 'A']						)



 def 				lowerCAmelCase_  (							self  :		List[Any]						):
       SCREAMING_SNAKE_CASE_     						=      Trie()
       trie.add('TOKEN]'						)
       trie.add('[SPECIAL_TOKEN]'						)
       self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'						)    ,    ['This is something ', '[SPECIAL_TOKEN]']						)



 def 				lowerCAmelCase_  (							self  :		Union[str, Any]						):
       SCREAMING_SNAKE_CASE_     						=      Trie()
       trie.add('A'						)
       trie.add('P'						)
       trie.add('[SPECIAL_TOKEN]'						)
       self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'						)    ,    ['This is something ', '[SPECIAL_TOKEN]']						)



 def 				lowerCAmelCase_  (							self  :		Tuple						):
       SCREAMING_SNAKE_CASE_     						=      Trie()
       trie.add('AB'						)
       trie.add('B'						)
       trie.add('C'						)
       self.assertEqual(trie.split('ABC'						)    ,    ['AB', 'C']						)



 def 				lowerCAmelCase_  (							self  :		Tuple						):
       SCREAMING_SNAKE_CASE_     						=      Trie()
       trie.add('ABC'						)
       trie.add('B'						)
       trie.add('CD'						)
       self.assertEqual(trie.split('ABCD'						)    ,    ['ABC', 'D']						)




 def 				lowerCAmelCase_  (							self  :		int						):
       # Even if the offsets are wrong, we necessarily output correct string
       # parts.
       SCREAMING_SNAKE_CASE_     						=      Trie()
       SCREAMING_SNAKE_CASE_     						=      trie.cut_text('ABC'    ,    [0, 0, 2, 1, 2, 3]						)
       self.assertEqual(_lowerCamelCase    ,    ['AB', 'C']						)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase__			:       Dict         =							logging.get_logger(__name__)

lowerCamelCase__			:       Any         =							{
    'facebook/s2t-small-librispeech-asr': (
        'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/config.json'
    ),
    # See all Speech2Text models at https://huggingface.co/models?filter=speech_to_text
}





class 	lowerCamelCase_							(				_SCREAMING_SNAKE_CASE ):


 '''simple docstring'''







 lowercase_       =   "speech_to_text"
 lowercase_       =   ["past_key_values"]
 lowercase_       =   {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}



 def __init__(							self  :		Union[str, Any]    ,    _lowerCAmelCase  :		Optional[int]=10_000    ,    _lowerCAmelCase  :		str=12    ,    _lowerCAmelCase  :		Tuple=2_048    ,    _lowerCAmelCase  :		str=4    ,    _lowerCAmelCase  :		Tuple=6    ,    _lowerCAmelCase  :		Optional[int]=2_048    ,    _lowerCAmelCase  :		Dict=4    ,    _lowerCAmelCase  :		List[Any]=0.0    ,    _lowerCAmelCase  :		int=0.0    ,    _lowerCAmelCase  :		Optional[int]=True    ,    _lowerCAmelCase  :		Union[str, Any]=True    ,    _lowerCAmelCase  :		Any="relu"    ,    _lowerCAmelCase  :		Any=256    ,    _lowerCAmelCase  :		List[str]=0.1    ,    _lowerCAmelCase  :		Optional[Any]=0.0    ,    _lowerCAmelCase  :		Optional[int]=0.0    ,    _lowerCAmelCase  :		Tuple=0.02    ,    _lowerCAmelCase  :		List[str]=2    ,    _lowerCAmelCase  :		Tuple=True    ,    _lowerCAmelCase  :		List[str]=1    ,    _lowerCAmelCase  :		Dict=0    ,    _lowerCAmelCase  :		Tuple=2    ,    _lowerCAmelCase  :		Optional[int]=6_000    ,    _lowerCAmelCase  :		Tuple=1_024    ,    _lowerCAmelCase  :		Any=2    ,    _lowerCAmelCase  :		str=(5, 5)    ,    _lowerCAmelCase  :		Optional[int]=1_024    ,    _lowerCAmelCase  :		List[Any]=80    ,    _lowerCAmelCase  :		List[Any]=1    ,    **_lowerCAmelCase  :		List[Any]    ,    ):
       SCREAMING_SNAKE_CASE_     						=      vocab_size
       SCREAMING_SNAKE_CASE_     						=      d_model
       SCREAMING_SNAKE_CASE_     						=      encoder_ffn_dim
       SCREAMING_SNAKE_CASE_     						=      encoder_layers
       SCREAMING_SNAKE_CASE_     						=      encoder_attention_heads
       SCREAMING_SNAKE_CASE_     						=      decoder_ffn_dim
       SCREAMING_SNAKE_CASE_     						=      decoder_layers
       SCREAMING_SNAKE_CASE_     						=      decoder_attention_heads
       SCREAMING_SNAKE_CASE_     						=      dropout
       SCREAMING_SNAKE_CASE_     						=      attention_dropout
       SCREAMING_SNAKE_CASE_     						=      activation_dropout
       SCREAMING_SNAKE_CASE_     						=      activation_function
       SCREAMING_SNAKE_CASE_     						=      init_std
       SCREAMING_SNAKE_CASE_     						=      encoder_layerdrop
       SCREAMING_SNAKE_CASE_     						=      decoder_layerdrop
       SCREAMING_SNAKE_CASE_     						=      use_cache
       SCREAMING_SNAKE_CASE_     						=      encoder_layers
       SCREAMING_SNAKE_CASE_     						=      scale_embedding  # scale factor will be sqrt(d_model) if True
       SCREAMING_SNAKE_CASE_     						=      max_source_positions
       SCREAMING_SNAKE_CASE_     						=      max_target_positions
       SCREAMING_SNAKE_CASE_     						=      num_conv_layers
       SCREAMING_SNAKE_CASE_     						=      list(_lowerCAmelCase						)
       SCREAMING_SNAKE_CASE_     						=      conv_channels
       SCREAMING_SNAKE_CASE_     						=      input_feat_per_channel
       SCREAMING_SNAKE_CASE_     						=      input_channels

       if len(self.conv_kernel_sizes						) != self.num_conv_layers:
             raise ValueError(
                 'Configuration for convolutional module is incorrect. '
                 'It is required that `len(config.conv_kernel_sizes)` == `config.num_conv_layers` '
                 F"but is `len(config.conv_kernel_sizes) = {len(self.conv_kernel_sizes						)}`, "
                 F"`config.num_conv_layers = {self.num_conv_layers}`."						)

       super().__init__(
           pad_token_id=_lowerCAmelCase    ,    bos_token_id=_lowerCAmelCase    ,    eos_token_id=_lowerCAmelCase    ,    is_encoder_decoder=_lowerCAmelCase    ,    decoder_start_token_id=_lowerCAmelCase    ,    **_lowerCAmelCase    ,    )