


import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE      :			List[str]      				=   get_tests_dir("fixtures/test_sentencepiece.model")

if is_sentencepiece_available():
 import sentencepiece as sp


_SCREAMING_SNAKE_CASE      :			Tuple      				=   5
_SCREAMING_SNAKE_CASE      :			Optional[int]      				=   10







@require_sentencepiece
@require_tokenizers
class   A__ (   snake_case_	, unittest.TestCase     ):







    """simple docstring"""

    __magic_name__        =		SpeechaTextTokenizer
    __magic_name__        =		False
    __magic_name__        =		True
    def 		a_      (	self   ):
         super().setUp()

         snake_case					=    sp.SentencePieceProcessor()
         spm_model.Load(__snake_case   )
         snake_case					=    ['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

         vocab += [spm_model.IdToPiece(id_   ) for id_ in range(len(__snake_case   )   )]
         snake_case					=    dict(zip(__snake_case      , range(len(__snake_case   )   )   )   )

         snake_case					=    Path(self.tmpdirname   )
         save_json(__snake_case      , save_dir / VOCAB_FILES_NAMES['''vocab_file''']   )
         if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
              copyfile(__snake_case      , save_dir / VOCAB_FILES_NAMES['''spm_file''']   )

         snake_case					=    SpeechaTextTokenizer.from_pretrained(self.tmpdirname   )
         tokenizer.save_pretrained(self.tmpdirname   )
    def 		a_      (	self   ):
         snake_case					=    '''<pad>'''
         snake_case					=    1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case   )      , __snake_case   )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case   )      , __snake_case   )
    def 		a_      (	self   ):
         snake_case					=    list(self.get_tokenizer().get_vocab().keys()   )

         self.assertEqual(vocab_keys[0]      , '''<s>'''   )
         self.assertEqual(vocab_keys[1]      , '''<pad>'''   )
         self.assertEqual(vocab_keys[-1]      , '''j'''   )
         self.assertEqual(len(__snake_case   )      , 1_0_0_1   )
    def 		a_      (	self   ):
         self.assertEqual(self.get_tokenizer().vocab_size      , 1_0_0_1   )
    def 		a_      (	self   ):
         snake_case					=    SpeechaTextTokenizer.from_pretrained(self.tmpdirname   )

         snake_case					=    tokenizer.tokenize('''This is a test'''   )
         self.assertListEqual(__snake_case      , ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']   )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(__snake_case   )      , [2_8_9, 5_0, 1_4, 1_7_4, 3_8_6]      , )

         snake_case					=    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''   )
         self.assertListEqual(
             __snake_case      , [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']      , )
         snake_case					=    tokenizer.convert_tokens_to_ids(__snake_case   )
         self.assertListEqual(__snake_case      , [1_2, 2_5, 8_8, 5_9, 2_8, 2_3, 1_1, 4, 6_0_6, 3_5_1, 3_5_1, 3_5_1, 7, 1_6, 7_0, 5_0, 7_6, 8_4, 1_0, 4, 8]   )

         snake_case					=    tokenizer.convert_ids_to_tokens(__snake_case   )
         self.assertListEqual(
             __snake_case      , [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']      , )




    @slow
    def 		a_      (	self   ):
         # fmt: off
         snake_case					=    {'''input_ids''': [[3_7_9_1, 7_9_7, 3_1, 1_1, 6_4, 7_9_7, 3_1, 2_4_2_9, 4_3_3, 1_2, 1_1_7_6, 1_2, 2_0, 7_8_6, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 3_2_3_8, 7_9_7, 3_1, 1_1, 3_5, 9_3, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_7, 6_1_0, 4_0, 6_2, 4_5_5, 6_5_7, 1_0_4_2, 1_2_3, 7_8_0, 1_7_7, 3_7, 3_0_9, 2_4_1, 1_2_9_8, 5_1_4, 2_0, 2_9_2, 2_7_3_7, 1_1_4, 2_4_6_9, 2_4_1, 8_5, 6_4, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 4, 5_0_9, 4_0_6, 4_2_3, 3_7, 6_0_1, 4, 7_7_7, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 2_8_4, 4, 3_3_8_8, 5_1_1, 4_5_9, 4, 3_5_5_5, 4_0, 3_2_1, 3_0_2, 7_0_5, 4, 3_3_8_8, 5_1_1, 5_8_3, 3_2_6, 5, 5, 5, 6_2, 3_3_1_0, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 3_2, 3_1, 8_5_3, 4_1_8, 6_4, 5_8_3, 5_1_1, 1_6_0_5, 6_2, 3_5, 9_3, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 1_5_2_1, 6_4, 5_8_3, 5_1_1, 5_1_9, 6_2, 2_0, 1_5_1_5, 7_6_4, 2_0, 1_4_9, 2_6_1, 5_6_2_5, 7_9_7_2, 2_0, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_9_2_5, 1_6_7_5, 1_1, 1_5, 8_0_2, 7_9_7_2, 5_7_6, 2_1_7, 1_5_0_8, 1_1, 3_5, 9_3, 1_2_5_3, 2_4_4_1, 1_5, 2_8_9, 6_5_2, 3_1, 4_1_6, 3_2_1, 3_8_4_2, 1_1_5, 4_0, 9_1_1, 8, 4_7_6, 6_1_9, 4, 3_8_0, 1_4_2, 4_2_3, 3_3_5, 2_4_0, 3_5, 9_3, 2_6_4, 8, 1_1, 3_3_5, 5_6_9, 4_2_0, 1_6_3, 5, 2], [2_6_0, 5_4_8, 5_2_8, 4_2_3, 2_0, 4_5_1, 2_0, 2_6_8_1, 1_1_5_3, 3_4_3_4, 2_0, 5_5_4_0, 3_7, 5_6_7, 1_2_6, 1_2_5_3, 2_4_4_1, 3_3_7_6, 4_4_9, 2_1_0, 4_3_1, 1_5_6_3, 1_7_7, 7_6_7, 5_5_4_0, 1_1, 1_2_0_3, 4_7_2, 1_1, 2_9_5_3, 6_8_5, 2_8_5, 3_6_4, 7_0_6, 1_1_5_3, 2_0, 6_7_9_9, 2_0, 2_8_6_9, 2_0, 4_4_6_4, 1_2_6, 4_0, 2_4_2_9, 2_0, 1_0_4_0, 8_6_6, 2_6_6_4, 4_1_8, 2_0, 3_1_8, 2_0, 1_7_2_6, 1_8_6, 2_0, 2_6_5, 5_2_2, 3_5, 9_3, 2_1_9_1, 4_6_3_4, 2_0, 1_0_4_0, 1_2, 6_7_9_9, 1_5, 2_2_8, 2_3_5_6, 1_4_2, 3_1, 1_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_7_5, 2_6_6_6, 6_8_4, 1_5_8_2, 1_1_7_6, 1_2, 6_2_7, 1_4_9, 6_1_9, 2_0, 4_9_0_2, 5_6_3, 1_1, 2_0, 1_4_9, 2_6_1, 3_4_2_0, 2_3_5_6, 1_7_4, 1_4_2, 4_7_1_4, 1_3_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=__snake_case      , model_name='''facebook/s2t-small-mustc-en-de-st'''      , revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad'''      , )







@require_sentencepiece
class   A__ (   unittest.TestCase     ):







    """simple docstring"""

    __magic_name__        =		'valhalla/s2t_mustc_multilinguial_medium'

    __magic_name__        =		'C\'est trop cool'
    __magic_name__        =		'Esto es genial'
    @classmethod
    def 		a_      (	cls   ):
         snake_case					=    SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name   )
         return cls
    def 		a_      (	self   ):
         self.assertEqual(self.tokenizer.lang_code_to_id['''pt''']      , 4   )
         self.assertEqual(self.tokenizer.lang_code_to_id['''ru''']      , 6   )
         self.assertEqual(self.tokenizer.lang_code_to_id['''it''']      , 9   )
         self.assertEqual(self.tokenizer.lang_code_to_id['''de''']      , 1_1   )
    def 		a_      (	self   ):
         self.assertEqual(self.tokenizer.vocab_size      , 1_0_0_0_0   )
    def 		a_      (	self   ):
         self.assertIn(__snake_case      , self.tokenizer.all_special_ids   )
         snake_case					=    [ES_CODE, 4, 1_6_0_1, 4_7, 7_6_4_7, 2]
         snake_case					=    self.tokenizer.decode(__snake_case      , skip_special_tokens=__snake_case   )
         snake_case					=    self.tokenizer.decode(generated_ids[1:]      , skip_special_tokens=__snake_case   )
         self.assertEqual(__snake_case      , __snake_case   )
         self.assertNotIn(self.tokenizer.eos_token      , __snake_case   )
    def 		a_      (	self   ):
         snake_case					=    '''fr'''
         snake_case					=    self.tokenizer(self.french_text   ).input_ids
         self.assertEqual(encoded[0]      , __snake_case   )
         self.assertEqual(encoded[-1]      , self.tokenizer.eos_token_id   )




    def 		a_      (	self   ):
         snake_case					=    '''fr'''
         self.assertListEqual(self.tokenizer.prefix_tokens      , [FR_CODE]   )

         snake_case					=    '''es'''
         self.assertListEqual(self.tokenizer.prefix_tokens      , [ES_CODE]   )








import copy
from collections import OrderedDict
from typing import Dict, Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


_SCREAMING_SNAKE_CASE      :			Dict      				=   logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE      :			Any      				=   {
    "facebook/detr-resnet-50": "https://huggingface.co/facebook/detr-resnet-50/resolve/main/config.json",
    # See all DETR models at https://huggingface.co/models?filter=detr
}

class   A__ (   snake_case__     ):







    """simple docstring"""

    __magic_name__        =		'detr'
    __magic_name__        =		['past_key_values']
    __magic_name__        =		{
        'hidden_size': 'd_model',
        'num_attention_heads': 'encoder_attention_heads',
    }
    def __init__(	self      , __snake_case=True      , __snake_case=None      , __snake_case=3      , __snake_case=1_0_0      , __snake_case=6      , __snake_case=2_0_4_8      , __snake_case=8      , __snake_case=6      , __snake_case=2_0_4_8      , __snake_case=8      , __snake_case=0.0      , __snake_case=0.0      , __snake_case=True      , __snake_case="relu"      , __snake_case=2_5_6      , __snake_case=0.1      , __snake_case=0.0      , __snake_case=0.0      , __snake_case=0.02      , __snake_case=1.0      , __snake_case=False      , __snake_case="sine"      , __snake_case="resnet50"      , __snake_case=True      , __snake_case=False      , __snake_case=1      , __snake_case=5      , __snake_case=2      , __snake_case=1      , __snake_case=1      , __snake_case=5      , __snake_case=2      , __snake_case=0.1      , **__snake_case      , ):
         if backbone_config is not None and use_timm_backbone:
              raise ValueError('''You can\'t specify both `backbone_config` and `use_timm_backbone`.'''   )

         if not use_timm_backbone:
              if backbone_config is None:
                   logger.info('''`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone.'''   )
                   snake_case					=    CONFIG_MAPPING['''resnet'''](out_features=['''stage4''']   )
              elif isinstance(__snake_case      , __snake_case   ):
                   snake_case					=    backbone_config.get('''model_type'''   )
                   snake_case					=    CONFIG_MAPPING[backbone_model_type]
                   snake_case					=    config_class.from_dict(__snake_case   )
              # set timm attributes to None
              snake_case			,  snake_case			,  snake_case					=    None, None, None

         snake_case					=    use_timm_backbone
         snake_case					=    backbone_config
         snake_case					=    num_channels
         snake_case					=    num_queries
         snake_case					=    d_model
         snake_case					=    encoder_ffn_dim
         snake_case					=    encoder_layers
         snake_case					=    encoder_attention_heads
         snake_case					=    decoder_ffn_dim
         snake_case					=    decoder_layers
         snake_case					=    decoder_attention_heads
         snake_case					=    dropout
         snake_case					=    attention_dropout
         snake_case					=    activation_dropout
         snake_case					=    activation_function
         snake_case					=    init_std
         snake_case					=    init_xavier_std
         snake_case					=    encoder_layerdrop
         snake_case					=    decoder_layerdrop
         snake_case					=    encoder_layers
         snake_case					=    auxiliary_loss
         snake_case					=    position_embedding_type
         snake_case					=    backbone
         snake_case					=    use_pretrained_backbone
         snake_case					=    dilation
         # Hungarian matcher
         snake_case					=    class_cost
         snake_case					=    bbox_cost
         snake_case					=    giou_cost
         # Loss coefficients
         snake_case					=    mask_loss_coefficient
         snake_case					=    dice_loss_coefficient
         snake_case					=    bbox_loss_coefficient
         snake_case					=    giou_loss_coefficient
         snake_case					=    eos_coefficient
         super().__init__(is_encoder_decoder=__snake_case      , **__snake_case   )
    @property
    def 		a_      (	self   ):
         return self.encoder_attention_heads
    @property
    def 		a_      (	self   ):
         return self.d_model
    @classmethod
    def 		a_      (	cls      , __snake_case      , **__snake_case   ):
         return cls(backbone_config=__snake_case      , **__snake_case   )




    def 		a_      (	self   ):
         snake_case					=    copy.deepcopy(self.__dict__   )
         if output["backbone_config"] is not None:
              snake_case					=    self.backbone_config.to_dict()
         snake_case					=    self.__class__.model_type
         return output






class   A__ (   snake_case__     ):







    """simple docstring"""

    __magic_name__        =		version.parse('1.11'     )
    @property
    def 		a_      (	self   ):
         return OrderedDict(
             [
                 ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
                 ('''pixel_mask''', {0: '''batch'''}),
             ]   )
    @property
    def 		a_      (	self   ):
         return 1E-5




    @property
    def 		a_      (	self   ):
         return 1_2




