

import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_            = logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_            = {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

SCREAMING_SNAKE_CASE_            = {
    'vocab_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'},
    'merges_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'},
}

SCREAMING_SNAKE_CASE_            = {
    'ctrl': 256,
}

SCREAMING_SNAKE_CASE_            = {
    'Pregnancy': 168629,
    'Christianity': 7675,
    'Explain': 106423,
    'Fitness': 63440,
    'Saving': 63163,
    'Ask': 27171,
    'Ass': 95985,
    'Joke': 163509,
    'Questions': 45622,
    'Thoughts': 49605,
    'Retail': 52342,
    'Feminism': 164338,
    'Writing': 11992,
    'Atheism': 192263,
    'Netflix': 48616,
    'Computing': 39639,
    'Opinion': 43213,
    'Alone': 44967,
    'Funny': 58917,
    'Gaming': 40358,
    'Human': 4088,
    'India': 1331,
    'Joker': 77138,
    'Diet': 36206,
    'Legal': 11859,
    'Norman': 4939,
    'Tip': 72689,
    'Weight': 52343,
    'Movies': 46273,
    'Running': 23425,
    'Science': 2090,
    'Horror': 37793,
    'Confession': 60572,
    'Finance': 12250,
    'Politics': 16360,
    'Scary': 191985,
    'Support': 12654,
    'Technologies': 32516,
    'Teenage': 66160,
    'Event': 32769,
    'Learned': 67460,
    'Notion': 182770,
    'Wikipedia': 37583,
    'Books': 6665,
    'Extract': 76050,
    'Confessions': 102701,
    'Conspiracy': 75932,
    'Links': 63674,
    'Narcissus': 150425,
    'Relationship': 54766,
    'Relationships': 134796,
    'Reviews': 41671,
    'News': 4256,
    'Translation': 26820,
    'multilingual': 128406,
}

def 							__SCREAMING_SNAKE_CASE							(			lowerCAmelCase:					Any	)				->    int:
    _UpperCAmelCase    :						List[Any]							=      set()
    _UpperCAmelCase    :						int							=      word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)	)
        _UpperCAmelCase    :						Optional[Any]							=      char

    _UpperCAmelCase    :						Optional[int]							=      set(lowerCAmelCase	)
    return pairs





class 			a					(		UpperCAmelCase			):
      _lowercase        =     VOCAB_FILES_NAMES
      _lowercase        =     PRETRAINED_VOCAB_FILES_MAP
      _lowercase        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase        =     CONTROL_CODES


      def __init__(    self  ,   A_  ,   A_  ,   A_="<unk>"  ,   **A_		):







          '''simple docstring'''


          super().__init__(unk_token=A_  ,   **A_		)

          with open(A_  ,   encoding="utf-8"		) as vocab_handle:
              _UpperCAmelCase    :						Optional[int]							=      json.load(A_		)
          _UpperCAmelCase    :						Tuple							=      {v: k for k, v in self.encoder.items()}
          with open(A_  ,   encoding="utf-8"		) as merges_handle:
              _UpperCAmelCase    :						Optional[int]							=      merges_handle.read().split("\n"		)[1:-1]
          _UpperCAmelCase    :						List[Any]							=      [tuple(merge.split()		) for merge in merges]
          _UpperCAmelCase    :						List[str]							=      dict(zip(A_  ,   range(len(A_		)		)		)		)
          _UpperCAmelCase    :						Dict							=      {}


      @property
      def     _UpperCAmelCase			(    self		):







          '''simple docstring'''


          return len(self.encoder		)


      def     _UpperCAmelCase			(    self		):







          '''simple docstring'''


          return dict(self.encoder  ,   **self.added_tokens_encoder		)


      def     _UpperCAmelCase			(    self  ,   A_		):







          '''simple docstring'''


          if token in self.cache:
              return self.cache[token]
          _UpperCAmelCase    :						int							=      tuple(A_		)
          _UpperCAmelCase    :						Union[str, Any]							=      tuple(list(word[:-1]		) + [word[-1] + "</w>"]		)
          _UpperCAmelCase    :						Optional[Any]							=      get_pairs(A_		)

          if not pairs:
              return token

          while True:
              _UpperCAmelCase    :						Tuple							=      min(A_  ,   key=lambda A_		: self.bpe_ranks.get(A_  ,   float("inf"		)		)		)
              if bigram not in self.bpe_ranks:
                  break
              _UpperCAmelCase    :						Union[str, Any]							=      bigram
              _UpperCAmelCase    :						int							=      []
              _UpperCAmelCase    :						Optional[Any]							=      0
              while i < len(A_		):
                  try:
                      _UpperCAmelCase    :						Optional[Any]							=      word.index(A_  ,   A_		)
                  except ValueError:
                      new_word.extend(word[i:]		)
                      break
                  else:
                      new_word.extend(word[i:j]		)
                      _UpperCAmelCase    :						Optional[Any]							=      j

                  if word[i] == first and i < len(A_		) - 1 and word[i + 1] == second:
                      new_word.append(first + second		)
                      i += 2
                  else:
                      new_word.append(word[i]		)
                      i += 1
              _UpperCAmelCase    :						str							=      tuple(A_		)
              _UpperCAmelCase    :						List[Any]							=      new_word
              if len(A_		) == 1:
                  break
              else:
                  _UpperCAmelCase    :						Dict							=      get_pairs(A_		)
          _UpperCAmelCase    :						Tuple							=      "@@ ".join(A_		)
          _UpperCAmelCase    :						List[Any]							=      word[:-4]
          _UpperCAmelCase    :						str							=      word
          return word


      def     _UpperCAmelCase			(    self  ,   A_		):







          '''simple docstring'''


          _UpperCAmelCase    :						Any							=      []

          _UpperCAmelCase    :						str							=      re.findall(R"\S+\n?"  ,   A_		)

          for token in words:
              split_tokens.extend(list(self.bpe(A_		).split(" "		)		)		)
          return split_tokens


      def     _UpperCAmelCase			(    self  ,   A_		):







          '''simple docstring'''


          return self.encoder.get(A_  ,   self.encoder.get(self.unk_token		)		)


      def     _UpperCAmelCase			(    self  ,   A_		):







          '''simple docstring'''


          return self.decoder.get(A_  ,   self.unk_token		)


      def     _UpperCAmelCase			(    self  ,   A_		):







          '''simple docstring'''


          _UpperCAmelCase    :						str							=      " ".join(A_		).replace("@@ "  ,   ""		).strip()
          return out_string





      def     _UpperCAmelCase			(    self  ,   A_  ,   A_ = None		):







          '''simple docstring'''


          if not os.path.isdir(A_		):
              logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
              return
          _UpperCAmelCase    :						Optional[Any]							=      os.path.join(
              A_  ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
          _UpperCAmelCase    :						List[Any]							=      os.path.join(
              A_  ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

          with open(A_  ,   "w"  ,   encoding="utf-8"		) as f:
              f.write(json.dumps(self.encoder  ,   indent=2  ,   sort_keys=A_  ,   ensure_ascii=A_		) + "\n"		)

          _UpperCAmelCase    :						Optional[int]							=      0
          with open(A_  ,   "w"  ,   encoding="utf-8"		) as writer:
              writer.write("#version: 0.2\n"		)
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,   key=lambda A_		: kv[1]		):
                  if index != token_index:
                      logger.warning(
                          f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                          " Please check that the tokenizer is not corrupted!"		)
                      _UpperCAmelCase    :						Tuple							=      token_index
                  writer.write(" ".join(A_		) + "\n"		)
                  index += 1

          return vocab_file, merge_file

      # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
      #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
      #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
      #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
      #     return ''.join(tokens_generated_so_far)





import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all image processors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...image_processing_utils import ImageProcessingMixin
from ...utils import CONFIG_NAME, IMAGE_PROCESSOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


SCREAMING_SNAKE_CASE_            = logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_            = OrderedDict(
    [
        ('align', 'EfficientNetImageProcessor'),
        ('beit', 'BeitImageProcessor'),
        ('bit', 'BitImageProcessor'),
        ('blip', 'BlipImageProcessor'),
        ('blip-2', 'BlipImageProcessor'),
        ('bridgetower', 'BridgeTowerImageProcessor'),
        ('chinese_clip', 'ChineseCLIPImageProcessor'),
        ('clip', 'CLIPImageProcessor'),
        ('clipseg', 'ViTImageProcessor'),
        ('conditional_detr', 'ConditionalDetrImageProcessor'),
        ('convnext', 'ConvNextImageProcessor'),
        ('convnextv2', 'ConvNextImageProcessor'),
        ('cvt', 'ConvNextImageProcessor'),
        ('data2vec-vision', 'BeitImageProcessor'),
        ('deformable_detr', 'DeformableDetrImageProcessor'),
        ('deit', 'DeiTImageProcessor'),
        ('deta', 'DetaImageProcessor'),
        ('detr', 'DetrImageProcessor'),
        ('dinat', 'ViTImageProcessor'),
        ('donut-swin', 'DonutImageProcessor'),
        ('dpt', 'DPTImageProcessor'),
        ('efficientformer', 'EfficientFormerImageProcessor'),
        ('efficientnet', 'EfficientNetImageProcessor'),
        ('flava', 'FlavaImageProcessor'),
        ('focalnet', 'BitImageProcessor'),
        ('git', 'CLIPImageProcessor'),
        ('glpn', 'GLPNImageProcessor'),
        ('groupvit', 'CLIPImageProcessor'),
        ('imagegpt', 'ImageGPTImageProcessor'),
        ('instructblip', 'BlipImageProcessor'),
        ('layoutlmv2', 'LayoutLMv2ImageProcessor'),
        ('layoutlmv3', 'LayoutLMv3ImageProcessor'),
        ('levit', 'LevitImageProcessor'),
        ('mask2former', 'Mask2FormerImageProcessor'),
        ('maskformer', 'MaskFormerImageProcessor'),
        ('mgp-str', 'ViTImageProcessor'),
        ('mobilenet_v1', 'MobileNetV1ImageProcessor'),
        ('mobilenet_v2', 'MobileNetV2ImageProcessor'),
        ('mobilevit', 'MobileViTImageProcessor'),
        ('mobilevit', 'MobileViTImageProcessor'),
        ('mobilevitv2', 'MobileViTImageProcessor'),
        ('nat', 'ViTImageProcessor'),
        ('oneformer', 'OneFormerImageProcessor'),
        ('owlvit', 'OwlViTImageProcessor'),
        ('perceiver', 'PerceiverImageProcessor'),
        ('pix2struct', 'Pix2StructImageProcessor'),
        ('poolformer', 'PoolFormerImageProcessor'),
        ('regnet', 'ConvNextImageProcessor'),
        ('resnet', 'ConvNextImageProcessor'),
        ('sam', 'SamImageProcessor'),
        ('segformer', 'SegformerImageProcessor'),
        ('swiftformer', 'ViTImageProcessor'),
        ('swin', 'ViTImageProcessor'),
        ('swin2sr', 'Swin2SRImageProcessor'),
        ('swinv2', 'ViTImageProcessor'),
        ('table-transformer', 'DetrImageProcessor'),
        ('timesformer', 'VideoMAEImageProcessor'),
        ('tvlt', 'TvltImageProcessor'),
        ('upernet', 'SegformerImageProcessor'),
        ('van', 'ConvNextImageProcessor'),
        ('videomae', 'VideoMAEImageProcessor'),
        ('vilt', 'ViltImageProcessor'),
        ('vit', 'ViTImageProcessor'),
        ('vit_hybrid', 'ViTHybridImageProcessor'),
        ('vit_mae', 'ViTImageProcessor'),
        ('vit_msn', 'ViTImageProcessor'),
        ('xclip', 'CLIPImageProcessor'),
        ('yolos', 'YolosImageProcessor'),
    ]
)

SCREAMING_SNAKE_CASE_            = _LazyAutoMapping(CONFIG_MAPPING_NAMES, IMAGE_PROCESSOR_MAPPING_NAMES)

def 							__SCREAMING_SNAKE_CASE							(			lowerCAmelCase:					str	)				->    Tuple:
				for module_name, extractors in IMAGE_PROCESSOR_MAPPING_NAMES.items():
								if class_name in extractors:
												_UpperCAmelCase    :						Union[str, Any]							=      model_type_to_module_name(lowerCAmelCase	)

												_UpperCAmelCase    :						Optional[int]							=      importlib.import_module(F'.{module_name}'					, "transformers.models"	)
												try:
																return getattr(lowerCAmelCase					, lowerCAmelCase	)
												except AttributeError:
																continue

				for _, extractor in IMAGE_PROCESSOR_MAPPING._extra_content.items():
								if getattr(lowerCAmelCase					, "__name__"					, lowerCAmelCase	) == class_name:
												return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
				_UpperCAmelCase    :						Any							=      importlib.import_module("transformers"	)
				if hasattr(lowerCAmelCase					, lowerCAmelCase	):
								return getattr(lowerCAmelCase					, lowerCAmelCase	)

				return None

def 							__SCREAMING_SNAKE_CASE							(			lowerCAmelCase:					Union[str, os.PathLike]					, lowerCAmelCase:					Optional[Union[str, os.PathLike]] = None					, lowerCAmelCase:					bool = False					, lowerCAmelCase:					bool = False					, lowerCAmelCase:					Optional[Dict[str, str]] = None					, lowerCAmelCase:					Optional[Union[bool, str]] = None					, lowerCAmelCase:					Optional[str] = None					, lowerCAmelCase:					bool = False					, **lowerCAmelCase:					List[Any]					, )				->    Any:
				_UpperCAmelCase    :						List[Any]							=      get_file_from_repo(
				    lowerCAmelCase					, lowerCAmelCase					, cache_dir=lowerCAmelCase					, force_download=lowerCAmelCase					, resume_download=lowerCAmelCase					, proxies=lowerCAmelCase					, use_auth_token=lowerCAmelCase					, revision=lowerCAmelCase					, local_files_only=lowerCAmelCase					, )
				if resolved_config_file is None:
								logger.info(
								    "Could not locate the image processor configuration file, will try to use the model config instead."	)
								return {}

				with open(lowerCAmelCase					, encoding="utf-8"	) as reader:
								return json.load(lowerCAmelCase	)





class 			a					:


						def __init__(    self		):







										'''simple docstring'''


										raise EnvironmentError(
										    "AutoImageProcessor is designed to be instantiated "
										    "using the `AutoImageProcessor.from_pretrained(pretrained_model_name_or_path)` method."		)


						@classmethod
						@replace_list_option_in_docstrings(A_		)
						def     _UpperCAmelCase			(    cls  ,   A_  ,   **A_		):







										'''simple docstring'''


										_UpperCAmelCase    :						str							=      kwargs.pop("config"  ,   A_		)
										_UpperCAmelCase    :						Dict							=      kwargs.pop("trust_remote_code"  ,   A_		)
										_UpperCAmelCase    :						str							=      True

										_UpperCAmelCase						,							_UpperCAmelCase    :						Tuple							=      ImageProcessingMixin.get_image_processor_dict(A_  ,   **A_		)
										_UpperCAmelCase    :						Any							=      config_dict.get("image_processor_type"  ,   A_		)
										_UpperCAmelCase    :						str							=      None
										if "AutoImageProcessor" in config_dict.get("auto_map"  ,   {}		):
														_UpperCAmelCase    :						int							=      config_dict["auto_map"]["AutoImageProcessor"]

										# If we still don't have the image processor class, check if we're loading from a previous feature extractor config
										# and if so, infer the image processor class from there.
										if image_processor_class is None and image_processor_auto_map is None:
														_UpperCAmelCase    :						Any							=      config_dict.pop("feature_extractor_type"  ,   A_		)
														if feature_extractor_class is not None:
																		logger.warning(
																		    "Could not find image processor class in the image processor config or the model config. Loading"
																		    " based on pattern matching with the model's feature extractor configuration."		)
																		_UpperCAmelCase    :						Optional[Any]							=      feature_extractor_class.replace("FeatureExtractor"  ,   "ImageProcessor"		)
														if "AutoFeatureExtractor" in config_dict.get("auto_map"  ,   {}		):
																		_UpperCAmelCase    :						Optional[Any]							=      config_dict["auto_map"]["AutoFeatureExtractor"]
																		_UpperCAmelCase    :						List[Any]							=      feature_extractor_auto_map.replace("FeatureExtractor"  ,   "ImageProcessor"		)
																		logger.warning(
																		    "Could not find image processor auto map in the image processor config or the model config."
																		    " Loading based on pattern matching with the model's feature extractor configuration."		)

        # If we don't find the image processor class in the image processor config, let's try the model config.
										if image_processor_class is None and image_processor_auto_map is None:
														if not isinstance(A_  ,   A_		):
																		_UpperCAmelCase    :						Any							=      AutoConfig.from_pretrained(A_  ,   **A_		)
														# It could be in `config.image_processor_type``
														_UpperCAmelCase    :						Optional[Any]							=      getattr(A_  ,   "image_processor_type"  ,   A_		)
														if hasattr(A_  ,   "auto_map"		) and "AutoImageProcessor" in config.auto_map:
																		_UpperCAmelCase    :						Any							=      config.auto_map["AutoImageProcessor"]

										if image_processor_class is not None:
														_UpperCAmelCase    :						List[str]							=      image_processor_class_from_name(A_		)

										_UpperCAmelCase    :						Optional[Any]							=      image_processor_auto_map is not None
										_UpperCAmelCase    :						Any							=      image_processor_class is not None or type(A_		) in IMAGE_PROCESSOR_MAPPING
										_UpperCAmelCase    :						List[Any]							=      resolve_trust_remote_code(
										    A_  ,   A_  ,   A_  ,   A_		)

										if has_remote_code and trust_remote_code:
														_UpperCAmelCase    :						Optional[int]							=      get_class_from_dynamic_module(
														    A_  ,   A_  ,   **A_		)
														_UpperCAmelCase    :						Optional[int]							=      kwargs.pop("code_revision"  ,   A_		)
														if os.path.isdir(A_		):
																		image_processor_class.register_for_auto_class()
														return image_processor_class.from_dict(A_  ,   **A_		)
										elif image_processor_class is not None:
														return image_processor_class.from_dict(A_  ,   **A_		)
										# Last try: we use the IMAGE_PROCESSOR_MAPPING.
										elif type(A_		) in IMAGE_PROCESSOR_MAPPING:
														_UpperCAmelCase    :						Optional[int]							=      IMAGE_PROCESSOR_MAPPING[type(A_		)]
														return image_processor_class.from_dict(A_  ,   **A_		)

										raise ValueError(
										    f'Unrecognized image processor in {pretrained_model_name_or_path}. Should have a '
										    f'`image_processor_type` key in its {IMAGE_PROCESSOR_NAME} of {CONFIG_NAME}, or one of the following '
										    f'`model_type` keys in its {CONFIG_NAME}: {", ".join(c for c in IMAGE_PROCESSOR_MAPPING_NAMES.keys()		)}'		)





						@staticmethod
						def     _UpperCAmelCase			(    A_  ,   A_		):







										'''simple docstring'''


										IMAGE_PROCESSOR_MAPPING.register(A_  ,   A_		)


