






"""simple docstring"""


import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


_UpperCamelCase    :   int   			=      logging.get_logger(__name__)

_UpperCamelCase    :   Optional[Any]   			=      OrderedDict(
    [
        ('audio-spectrogram-transformer', 'ASTFeatureExtractor'),
        ('beit', 'BeitFeatureExtractor'),
        ('chinese_clip', 'ChineseCLIPFeatureExtractor'),
        ('clap', 'ClapFeatureExtractor'),
        ('clip', 'CLIPFeatureExtractor'),
        ('clipseg', 'ViTFeatureExtractor'),
        ('conditional_detr', 'ConditionalDetrFeatureExtractor'),
        ('convnext', 'ConvNextFeatureExtractor'),
        ('cvt', 'ConvNextFeatureExtractor'),
        ('data2vec-audio', 'Wav2Vec2FeatureExtractor'),
        ('data2vec-vision', 'BeitFeatureExtractor'),
        ('deformable_detr', 'DeformableDetrFeatureExtractor'),
        ('deit', 'DeiTFeatureExtractor'),
        ('detr', 'DetrFeatureExtractor'),
        ('dinat', 'ViTFeatureExtractor'),
        ('donut-swin', 'DonutFeatureExtractor'),
        ('dpt', 'DPTFeatureExtractor'),
        ('encodec', 'EncodecFeatureExtractor'),
        ('flava', 'FlavaFeatureExtractor'),
        ('glpn', 'GLPNFeatureExtractor'),
        ('groupvit', 'CLIPFeatureExtractor'),
        ('hubert', 'Wav2Vec2FeatureExtractor'),
        ('imagegpt', 'ImageGPTFeatureExtractor'),
        ('layoutlmv2', 'LayoutLMv2FeatureExtractor'),
        ('layoutlmv3', 'LayoutLMv3FeatureExtractor'),
        ('levit', 'LevitFeatureExtractor'),
        ('maskformer', 'MaskFormerFeatureExtractor'),
        ('mctct', 'MCTCTFeatureExtractor'),
        ('mobilenet_v1', 'MobileNetV1FeatureExtractor'),
        ('mobilenet_v2', 'MobileNetV2FeatureExtractor'),
        ('mobilevit', 'MobileViTFeatureExtractor'),
        ('nat', 'ViTFeatureExtractor'),
        ('owlvit', 'OwlViTFeatureExtractor'),
        ('perceiver', 'PerceiverFeatureExtractor'),
        ('poolformer', 'PoolFormerFeatureExtractor'),
        ('regnet', 'ConvNextFeatureExtractor'),
        ('resnet', 'ConvNextFeatureExtractor'),
        ('segformer', 'SegformerFeatureExtractor'),
        ('sew', 'Wav2Vec2FeatureExtractor'),
        ('sew-d', 'Wav2Vec2FeatureExtractor'),
        ('speech_to_text', 'Speech2TextFeatureExtractor'),
        ('speecht5', 'SpeechT5FeatureExtractor'),
        ('swiftformer', 'ViTFeatureExtractor'),
        ('swin', 'ViTFeatureExtractor'),
        ('swinv2', 'ViTFeatureExtractor'),
        ('table-transformer', 'DetrFeatureExtractor'),
        ('timesformer', 'VideoMAEFeatureExtractor'),
        ('tvlt', 'TvltFeatureExtractor'),
        ('unispeech', 'Wav2Vec2FeatureExtractor'),
        ('unispeech-sat', 'Wav2Vec2FeatureExtractor'),
        ('van', 'ConvNextFeatureExtractor'),
        ('videomae', 'VideoMAEFeatureExtractor'),
        ('vilt', 'ViltFeatureExtractor'),
        ('vit', 'ViTFeatureExtractor'),
        ('vit_mae', 'ViTFeatureExtractor'),
        ('vit_msn', 'ViTFeatureExtractor'),
        ('wav2vec2', 'Wav2Vec2FeatureExtractor'),
        ('wav2vec2-conformer', 'Wav2Vec2FeatureExtractor'),
        ('wavlm', 'Wav2Vec2FeatureExtractor'),
        ('whisper', 'WhisperFeatureExtractor'),
        ('xclip', 'CLIPFeatureExtractor'),
        ('yolos', 'YolosFeatureExtractor'),
    ]
)

_UpperCamelCase    :   int   			=      _LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)




def       snake_case  (A_				:str				):







  '''simple docstring'''

  for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
    if class_name in extractors:
      a					:      Optional[int]    =       model_type_to_module_name(lowerCamelCase__				)

      a					:      List[Any]    =       importlib.import_module(f'''.{module_name}'''     ,      'transformers.models'				)
      try:
        return getattr(lowerCamelCase__     ,      lowerCamelCase__				)
      except AttributeError:
        continue

  for _, extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.items():
    if getattr(lowerCamelCase__     ,      '__name__'     ,      lowerCamelCase__				) == class_name:
      return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
  a					:      str    =       importlib.import_module('transformers'				)
  if hasattr(lowerCamelCase__     ,      lowerCamelCase__				):
    return getattr(lowerCamelCase__     ,      lowerCamelCase__				)

  return None




def       snake_case  (A_				:Union[str, os.PathLike]     ,      A_				:Optional[Union[str, os.PathLike]] = None     ,      A_				:bool = False     ,      A_				:bool = False     ,      A_				:Optional[Dict[str, str]] = None     ,      A_				:Optional[Union[bool, str]] = None     ,      A_				:Optional[str] = None     ,      A_				:bool = False     ,      **A_				:Optional[int]     ,      ):







  '''simple docstring'''

  a					:      Dict    =       get_file_from_repo(
      lowerCamelCase__     ,      lowerCamelCase__     ,      cache_dir=lowerCamelCase__     ,      force_download=lowerCamelCase__     ,      resume_download=lowerCamelCase__     ,      proxies=lowerCamelCase__     ,      use_auth_token=lowerCamelCase__     ,      revision=lowerCamelCase__     ,      local_files_only=lowerCamelCase__     ,      )
  if resolved_config_file is None:
    logger.info(
        'Could not locate the feature extractor configuration file, will try to use the model config instead.'				)
    return {}

  with open(lowerCamelCase__     ,      encoding='utf-8'				) as reader:
    return json.load(lowerCamelCase__				)






class 						snake_case  :





    def __init__(							self     : Optional[int]      ):


      '''simple docstring'''




      raise EnvironmentError(
          'AutoFeatureExtractor is designed to be instantiated '
          'using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method.'      )





    @classmethod
    @replace_list_option_in_docstrings(A      )
    def     lowerCamelCase__				(							cls     : Union[str, Any]    ,		A     : List[str]    ,		**A     : int      ):


      '''simple docstring'''




      a					:      str    =       kwargs.pop('config'    ,		A      )
      a					:      Dict    =       kwargs.pop('trust_remote_code'    ,		A      )
      a					:      List[str]    =       True

      a,					a					:      Tuple    =       FeatureExtractionMixin.get_feature_extractor_dict(A    ,		**A      )
      a					:      List[Any]    =       config_dict.get('feature_extractor_type'    ,		A      )
      a					:      Tuple    =       None
      if "AutoFeatureExtractor" in config_dict.get('auto_map'    ,		{}      ):
        a					:      Union[str, Any]    =       config_dict['auto_map']['AutoFeatureExtractor']

      # If we don't find the feature extractor class in the feature extractor config, let's try the model config.
      if feature_extractor_class is None and feature_extractor_auto_map is None:
        if not isinstance(A    ,		A      ):
          a					:      List[str]    =       AutoConfig.from_pretrained(A    ,		**A      )
        # It could be in `config.feature_extractor_type``
        a					:      str    =       getattr(A    ,		'feature_extractor_type'    ,		A      )
        if hasattr(A    ,		'auto_map'      ) and "AutoFeatureExtractor" in config.auto_map:
          a					:      Optional[int]    =       config.auto_map['AutoFeatureExtractor']

      if feature_extractor_class is not None:
        a					:      Any    =       feature_extractor_class_from_name(A      )

      a					:      Dict    =       feature_extractor_auto_map is not None
      a					:      int    =       feature_extractor_class is not None or type(A      ) in FEATURE_EXTRACTOR_MAPPING
      a					:      str    =       resolve_trust_remote_code(
          A    ,		A    ,		A    ,		A      )

      if has_remote_code and trust_remote_code:
        a					:      Any    =       get_class_from_dynamic_module(
            A    ,		A    ,		**A      )
        a					:      List[str]    =       kwargs.pop('code_revision'    ,		A      )
        if os.path.isdir(A      ):
          feature_extractor_class.register_for_auto_class()
        return feature_extractor_class.from_dict(A    ,		**A      )
      elif feature_extractor_class is not None:
        return feature_extractor_class.from_dict(A    ,		**A      )
      # Last try: we use the FEATURE_EXTRACTOR_MAPPING.
      elif type(A      ) in FEATURE_EXTRACTOR_MAPPING:
        a					:      Dict    =       FEATURE_EXTRACTOR_MAPPING[type(A      )]
        return feature_extractor_class.from_dict(A    ,		**A      )

      raise ValueError(
          F'''Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a '''
          F'''`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following '''
          F'''`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES.keys()      )}'''      )





    @staticmethod
    def     lowerCamelCase__				(							A     : List[str]    ,		A     : List[Any]      ):


      '''simple docstring'''




      FEATURE_EXTRACTOR_MAPPING.register(A    ,		A      )












"""simple docstring"""


import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase    :   int   			=						get_tests_dir('fixtures/test_sentencepiece_bpe.model')






class 						snake_case  (  UpperCAmelCase       ,			unittest.TestCase	):
    __magic_name__   					=					BartphoTokenizer
    __magic_name__   					=					False
    __magic_name__   					=					True





    def     lowerCamelCase__				(							self     : List[Any]      ):


      '''simple docstring'''




      super().setUp()

      a					:      Any    =       ['▁This', '▁is', '▁a', '▁t', 'est']
      a					:      List[Any]    =       dict(zip(A    ,		range(len(A      )      )      )      )
      a					:      int    =       {'unk_token': '<unk>'}

      a					:      Tuple    =       os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['monolingual_vocab_file']      )
      with open(self.monolingual_vocab_file    ,		'w'    ,		encoding='utf-8'      ) as fp:
        for token in vocab_tokens:
          fp.write(F'''{token} {vocab_tokens[token]}\n'''      )

      a					:      Optional[int]    =       BartphoTokenizer(A    ,		self.monolingual_vocab_file    ,		**self.special_tokens_map      )
      tokenizer.save_pretrained(self.tmpdirname      )





    def     lowerCamelCase__				(							self     : Dict    ,		**A     : str      ):


      '''simple docstring'''




      kwargs.update(self.special_tokens_map      )
      return BartphoTokenizer.from_pretrained(self.tmpdirname    ,		**A      )





    def     lowerCamelCase__				(							self     : Optional[int]    ,		A     : Dict      ):


      '''simple docstring'''




      a					:      Tuple    =       'This is a là test'
      a					:      List[Any]    =       'This is a<unk><unk> test'
      return input_text, output_text





    def     lowerCamelCase__				(							self     : Optional[Any]      ):


      '''simple docstring'''




      a					:      Tuple    =       BartphoTokenizer(A    ,		self.monolingual_vocab_file    ,		**self.special_tokens_map      )
      a					:      int    =       'This is a là test'
      a					:      int    =       '▁This ▁is ▁a ▁l à ▁t est'.split()
      a					:      str    =       tokenizer.tokenize(A      )
      self.assertListEqual(A    ,		A      )

      a					:      Union[str, Any]    =       tokens + [tokenizer.unk_token]
      a					:      Dict    =       [4, 5, 6, 3, 3, 7, 8, 3]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(A      )    ,		A      )




