

import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_            = logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_            = '▁'

SCREAMING_SNAKE_CASE_            = {'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

SCREAMING_SNAKE_CASE_            = {
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

SCREAMING_SNAKE_CASE_            = {
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

SCREAMING_SNAKE_CASE_            = {
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

SCREAMING_SNAKE_CASE_            = {
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}

class 			a					(		UpperCAmelCase			):
						_lowercase        =     ["input_ids"]

						_lowercase        =     VOCAB_FILES_NAMES
						_lowercase        =     PRETRAINED_INIT_CONFIGURATION
						_lowercase        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_lowercase        =     PRETRAINED_VOCAB_FILES_MAP
						_lowercase        =     RESOURCE_FILES_NAMES


						def __init__(    self  ,   A_  ,   A_=None  ,   A_=False  ,   A_="utf8"  ,   A_="[UNK]"  ,   A_="[SEP]"  ,   A_="[PAD]"  ,   A_="[CLS]"  ,   A_="[MASK]"  ,   A_ = None  ,   **A_  ,   ):







										'''simple docstring'''


										_UpperCAmelCase    :						List[str]							=      {} if sp_model_kwargs is None else sp_model_kwargs
										super().__init__(
										    do_lower_case=A_  ,   unk_token=A_  ,   sep_token=A_  ,   pad_token=A_  ,   cls_token=A_  ,   mask_token=A_  ,   vocab_file=A_  ,   encoding=A_  ,   sp_model_kwargs=self.sp_model_kwargs  ,   **A_  ,   )
										_UpperCAmelCase    :						List[Any]							=      do_lower_case
										_UpperCAmelCase    :						Dict							=      sentencepiece_model_ckpt
										_UpperCAmelCase    :						Tuple							=      spm.SentencePieceProcessor(**self.sp_model_kwargs		)
										self.sp_model.Load(A_		)

										# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
										if vocab_file is not None:
														_UpperCAmelCase    :						List[str]							=      self.load_vocab(filepath=A_		)
										else:
														_UpperCAmelCase    :						Optional[int]							=      {self.sp_model.id_to_piece(A_		): id for id in range(self.sp_model.get_piece_size()		)}
										_UpperCAmelCase    :						List[Any]							=      {v: k for k, v in self.vocab.items()}


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										if text is None:
														return None

										_UpperCAmelCase    :						Tuple							=      self.tokenize(A_		)
										_UpperCAmelCase						,							_UpperCAmelCase    :						str							=      "", []

										for i, ch in enumerate(A_		):
														if ch in self.SP_CHAR_MAPPING:
																		_UpperCAmelCase    :						Dict							=      self.SP_CHAR_MAPPING.get(A_		)
														else:
																		_UpperCAmelCase    :						List[Any]							=      unicodedata.normalize("NFKC"  ,   A_		)
														if self.is_whitespace(A_		):
																		continue
														normalized_text += ch
														char_mapping.extend([i] * len(A_		)		)

										_UpperCAmelCase						,							_UpperCAmelCase						,							_UpperCAmelCase    :						Union[str, Any]							=      normalized_text, [], 0

										if self.do_lower_case:
														_UpperCAmelCase    :						Optional[int]							=      text.lower()

										for token in split_tokens:
														if token[:1] == "▁":
																		_UpperCAmelCase    :						List[str]							=      token[1:]
														_UpperCAmelCase    :						str							=      text[offset:].index(A_		) + offset
														_UpperCAmelCase    :						str							=      start + len(A_		)

														token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)		)
														_UpperCAmelCase    :						List[str]							=      end
										return token_mapping


						@property
						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										return len(self.vocab		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										return dict(self.vocab  ,   **self.added_tokens_encoder		)


						def __getstate__(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						Any							=      self.__dict__.copy()
										_UpperCAmelCase    :						List[str]							=      None
										return state


						def __setstate__(    self  ,   A_		):







										'''simple docstring'''


										_UpperCAmelCase    :						Dict							=      d

										# for backward compatibility
										if not hasattr(self  ,   "sp_model_kwargs"		):
														_UpperCAmelCase    :						Optional[int]							=      {}

										_UpperCAmelCase    :						List[str]							=      spm.SentencePieceProcessor(**self.sp_model_kwargs		)
										self.sp_model.Load(self.sentencepiece_model_ckpt		)


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										return "".join((self.SP_CHAR_MAPPING.get(A_  ,   A_		) for c in text)		)


						def     _UpperCAmelCase			(    self  ,   A_  ,   A_=False  ,   A_=64  ,   A_=0.1		):







										'''simple docstring'''


										if self.sp_model_kwargs.get("enable_sampling"		) is True:
														_UpperCAmelCase    :						List[str]							=      True
										if self.sp_model_kwargs.get("alpha"		) is not None:
														_UpperCAmelCase    :						Tuple							=      self.sp_model_kwargs.get("alpha"		)
										if self.sp_model_kwargs.get("nbest_size"		) is not None:
														_UpperCAmelCase    :						Union[str, Any]							=      self.sp_model_kwargs.get("nbest_size"		)

										if not enable_sampling:
														_UpperCAmelCase    :						Optional[Any]							=      self.sp_model.EncodeAsPieces(A_		)
										else:
														_UpperCAmelCase    :						int							=      self.sp_model.SampleEncodeAsPieces(A_  ,   A_  ,   A_		)
										_UpperCAmelCase    :						List[Any]							=      []
										for pi, piece in enumerate(A_		):
														if piece == SPIECE_UNDERLINE:
																		if not pieces[pi + 1].startswith(A_		) and pi != 0:
																						new_pieces.append(A_		)
																						continue
																		else:
																						continue
														_UpperCAmelCase    :						Tuple							=      0
														for i, chunk in enumerate(A_		):
																		if chunk == SPIECE_UNDERLINE:
																						continue
																		if self.is_ch_char(A_		) or self.is_punct(A_		):
																						if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																										new_pieces.append(piece[lst_i:i]		)
																						new_pieces.append(A_		)
																						_UpperCAmelCase    :						Dict							=      i + 1
																		elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																						if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																										new_pieces.append(piece[lst_i:i]		)
																						_UpperCAmelCase    :						List[Any]							=      i
																		elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																						if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																										new_pieces.append(piece[lst_i:i]		)
																						_UpperCAmelCase    :						Optional[Any]							=      i
														if len(A_		) > lst_i:
																		new_pieces.append(piece[lst_i:]		)
										return new_pieces


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										_UpperCAmelCase    :						Union[str, Any]							=      "".join(A_		).replace(A_  ,   " "		).strip()
										return out_string


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										_UpperCAmelCase    :						List[str]							=      self.convert_ids_to_tokens(A_		)
										_UpperCAmelCase    :						Optional[int]							=      "".join(A_		).replace(A_  ,   " "		).strip()
										return out_string


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										return self.vocab.get(A_  ,   self.vocab.get(self.unk_token		)		)


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										return self.reverse_vocab.get(A_  ,   self.unk_token		)


						def     _UpperCAmelCase			(    self  ,   A_  ,   A_=None		):







										'''simple docstring'''


										if token_ids_a is None:
														return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										_UpperCAmelCase    :						str							=      [self.cls_token_id]
										_UpperCAmelCase    :						int							=      [self.sep_token_id]
										return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


						def     _UpperCAmelCase			(    self  ,   A_  ,   A_=None		):







										'''simple docstring'''


										if offset_mapping_a is None:
														return [(0, 0)] + offset_mapping_a + [(0, 0)]

										return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


						def     _UpperCAmelCase			(    self  ,   A_  ,   A_=None  ,   A_=False		):







										'''simple docstring'''


										if already_has_special_tokens:
														if token_ids_a is not None:
																		raise ValueError(
																		    "You should not supply a second sequence if the provided sequence of "
																		    "ids is already formatted with special tokens for the model."		)
														return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

										if token_ids_a is not None:
														return [1] + ([0] * len(A_		)) + [1, 1] + ([0] * len(A_		)) + [1]
										return [1] + ([0] * len(A_		)) + [1]


						def     _UpperCAmelCase			(    self  ,   A_  ,   A_ = None		):







										'''simple docstring'''


										if token_ids_a is None:
														# [CLS] X [SEP]
														return (len(A_		) + 2) * [0]

										# [CLS] A [SEP] [SEP] B [SEP]
										return [0] * (len(A_		) + 1) + [1] * (len(A_		) + 3)


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										if "\u4e00" <= char <= "\u9fff":
														return True
										return False


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										if ("a" <= char <= "z") or ("A" <= char <= "Z"):
														return True
										return False


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										if char in ",;:.?!~，；：。？！《》【】":
														return True
										return False


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										if char == " " or char == "\t" or char == "\n" or char == "\r":
														return True
										if len(A_		) == 1:
														_UpperCAmelCase    :						List[Any]							=      unicodedata.category(A_		)
														if cat == "Zs":
																		return True
										return False


						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										_UpperCAmelCase    :						Dict							=      {}
										with io.open(A_  ,   "r"  ,   encoding="utf-8"		) as f:
														for index, line in enumerate(A_		):
																		_UpperCAmelCase    :						Optional[int]							=      line.rstrip("\n"		)
																		_UpperCAmelCase    :						str							=      int(A_		)

										return token_to_idx





						def     _UpperCAmelCase			(    self  ,   A_  ,   A_ = None		):







										'''simple docstring'''


										_UpperCAmelCase    :						Optional[Any]							=      0
										if os.path.isdir(A_		):
														_UpperCAmelCase    :						List[str]							=      os.path.join(
														    A_  ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
										else:
														_UpperCAmelCase    :						Any							=      (filename_prefix + "-" if filename_prefix else "") + save_directory
										with open(A_  ,   "w"  ,   encoding="utf-8"		) as writer:
														for token, token_index in sorted(self.vocab.items()  ,   key=lambda A_		: kv[1]		):
																		if index != token_index:
																						logger.warning(
																						    f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																						    " Please check that the vocabulary is not corrupted!"		)
																						_UpperCAmelCase    :						List[str]							=      token_index
																		writer.write(token + "\n"		)
																		index += 1

										_UpperCAmelCase    :						Any							=      os.path.join(A_  ,   "sentencepiece.bpe.model"		)
										with open(A_  ,   "wb"		) as fi:
														_UpperCAmelCase    :						Any							=      self.sp_model.serialized_model_proto()
														fi.write(A_		)

										return (vocab_file,)





import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
SCREAMING_SNAKE_CASE_            = 'src/transformers'


# This is to make sure the transformers module imported is the one in the repo.
SCREAMING_SNAKE_CASE_            = direct_transformers_import(PATH_TO_TRANSFORMERS)

SCREAMING_SNAKE_CASE_            = transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
SCREAMING_SNAKE_CASE_            = re.compile(R'\[(.+?)\]\((https://huggingface\.co/.+?)\)')


SCREAMING_SNAKE_CASE_            = {
    'DecisionTransformerConfig',
    'EncoderDecoderConfig',
    'MusicgenConfig',
    'RagConfig',
    'SpeechEncoderDecoderConfig',
    'TimmBackboneConfig',
    'VisionEncoderDecoderConfig',
    'VisionTextDualEncoderConfig',
    'LlamaConfig',
}

def 							__SCREAMING_SNAKE_CASE							(			lowerCAmelCase:					Dict	)				->    Union[str, Any]:
				_UpperCAmelCase    :						List[Any]							=      None

				# source code of `config_class`
				_UpperCAmelCase    :						int							=      inspect.getsource(lowerCAmelCase	)
				_UpperCAmelCase    :						Tuple							=      _re_checkpoint.findall(lowerCAmelCase	)

				# Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
				# For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
				for ckpt_name, ckpt_link in checkpoints:
								# allow the link to end with `/`
								if ckpt_link.endswith("/"	):
												_UpperCAmelCase    :						Dict							=      ckpt_link[:-1]

								# verify the checkpoint name corresponds to the checkpoint link
								_UpperCAmelCase    :						Any							=      F'https://huggingface.co/{ckpt_name}'
								if ckpt_link == ckpt_link_from_name:
												_UpperCAmelCase    :						Union[str, Any]							=      ckpt_name
												break

				return checkpoint

def 							__SCREAMING_SNAKE_CASE							(			)				->    Dict:
				_UpperCAmelCase    :						Any							=      []

				for config_class in list(CONFIG_MAPPING.values()	):
								# Skip deprecated models
								if "models.deprecated" in config_class.__module__:
												continue
								_UpperCAmelCase    :						int							=      get_checkpoint_from_config_class(lowerCAmelCase	)

								_UpperCAmelCase    :						Tuple							=      config_class.__name__
								if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
												configs_without_checkpoint.append(lowerCAmelCase	)

				if len(lowerCAmelCase	) > 0:
								_UpperCAmelCase    :						int							=      "\n".join(sorted(lowerCAmelCase	)	)
								raise ValueError(F'The following configurations don\'t contain any valid checkpoint:\n{message}'	)


if __name__ == "__main__":
					check_config_docstrings_have_checkpoints()


