





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =      '''▁'''

_SCREAMING_SNAKE_CASE   =      {'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

_SCREAMING_SNAKE_CASE   =      {
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

_SCREAMING_SNAKE_CASE   =      {
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

_SCREAMING_SNAKE_CASE   =      {
    '''ernie-m-base''': 5_1_4,
    '''ernie-m-large''': 5_1_4,
}

_SCREAMING_SNAKE_CASE   =      {
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	a      :			Any     =    ["input_ids"]

	a      :			Dict     =    VOCAB_FILES_NAMES
	a      :			List[str]     =    PRETRAINED_INIT_CONFIGURATION
	a      :			List[str]     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a      :			str     =    PRETRAINED_VOCAB_FILES_MAP
	a      :			str     =    RESOURCE_FILES_NAMES


	def __init__(self       ,_lowerCamelCase       ,_lowerCamelCase=None       ,_lowerCamelCase=False       ,_lowerCamelCase="utf8"       ,_lowerCamelCase="[UNK]"       ,_lowerCamelCase="[SEP]"       ,_lowerCamelCase="[PAD]"       ,_lowerCamelCase="[CLS]"       ,_lowerCamelCase="[MASK]"       ,_lowerCamelCase = None       ,**_lowerCamelCase       ,)							->							None:







				'''simple docstring'''





				__lowercase			  =							{} if sp_model_kwargs is None else sp_model_kwargs
				super().__init__(
				    do_lower_case=lowercase_       ,unk_token=lowercase_       ,sep_token=lowercase_       ,pad_token=lowercase_       ,cls_token=lowercase_       ,mask_token=lowercase_       ,vocab_file=lowercase_       ,encoding=lowercase_       ,sp_model_kwargs=self.sp_model_kwargs       ,**lowercase_       ,)
				__lowercase			  =							do_lower_case
				__lowercase			  =							sentencepiece_model_ckpt
				__lowercase			  =							spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.Load(lowercase_	)

				# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
				if vocab_file is not None:
							__lowercase			  =							self.load_vocab(filepath=lowercase_	)
				else:
							__lowercase			  =							{self.sp_model.id_to_piece(lowercase_	): id for id in range(self.sp_model.get_piece_size()	)}
				__lowercase			  =							{v: k for k, v in self.vocab.items()}


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Optional[Any]:







				'''simple docstring'''





				if text is None:
							return None

				__lowercase			  =							self.tokenize(lowercase_	)
				__lowercase  ,						__lowercase			  =							'''''', []

				for i, ch in enumerate(lowercase_	):
							if ch in self.SP_CHAR_MAPPING:
										__lowercase			  =							self.SP_CHAR_MAPPING.get(lowercase_	)
							else:
										__lowercase			  =							unicodedata.normalize('''NFKC'''       ,lowercase_	)
							if self.is_whitespace(lowercase_	):
										continue
							normalized_text += ch
							char_mapping.extend([i] * len(lowercase_	)	)

				__lowercase  ,						__lowercase  ,						__lowercase			  =							normalized_text, [], 0

				if self.do_lower_case:
							__lowercase			  =							text.lower()

				for token in split_tokens:
							if token[:1] == "▁":
										__lowercase			  =							token[1:]
							__lowercase			  =							text[offset:].index(lowercase_	) + offset
							__lowercase			  =							start + len(lowercase_	)

							token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)	)
							__lowercase			  =							end
				return token_mapping


	@property
	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				return len(self.vocab	)


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				return dict(self.vocab       ,**self.added_tokens_encoder	)


	def __getstate__(self	)							->							Optional[int]:







				'''simple docstring'''





				__lowercase			  =							self.__dict__.copy()
				__lowercase			  =							None
				return state


	def __setstate__(self       ,_lowerCamelCase	)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							d

				# for backward compatibility
				if not hasattr(self       ,'''sp_model_kwargs'''	):
							__lowercase			  =							{}

				__lowercase			  =							spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.Load(self.sentencepiece_model_ckpt	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Optional[int]:







				'''simple docstring'''





				return "".join((self.SP_CHAR_MAPPING.get(lowercase_       ,lowercase_	) for c in text)	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase=False       ,_lowerCamelCase=64       ,_lowerCamelCase=0.1	)							->							str:







				'''simple docstring'''





				if self.sp_model_kwargs.get('''enable_sampling'''	) is True:
							__lowercase			  =							True
				if self.sp_model_kwargs.get('''alpha'''	) is not None:
							__lowercase			  =							self.sp_model_kwargs.get('''alpha'''	)
				if self.sp_model_kwargs.get('''nbest_size'''	) is not None:
							__lowercase			  =							self.sp_model_kwargs.get('''nbest_size'''	)

				if not enable_sampling:
							__lowercase			  =							self.sp_model.EncodeAsPieces(lowercase_	)
				else:
							__lowercase			  =							self.sp_model.SampleEncodeAsPieces(lowercase_       ,lowercase_       ,lowercase_	)
				__lowercase			  =							[]
				for pi, piece in enumerate(lowercase_	):
							if piece == SPIECE_UNDERLINE:
										if not pieces[pi + 1].startswith(lowercase_	) and pi != 0:
													new_pieces.append(lowercase_	)
													continue
										else:
													continue
							__lowercase			  =							0
							for i, chunk in enumerate(lowercase_	):
										if chunk == SPIECE_UNDERLINE:
													continue
										if self.is_ch_char(lowercase_	) or self.is_punct(lowercase_	):
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i]	)
													new_pieces.append(lowercase_	)
													__lowercase			  =							i + 1
										elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i]	)
													__lowercase			  =							i
										elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i]	)
													__lowercase			  =							i
							if len(lowercase_	) > lst_i:
										new_pieces.append(piece[lst_i:]	)
				return new_pieces


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							''''''.join(lowercase_	).replace(lowercase_       ,''' '''	).strip()
				return out_string


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Any:







				'''simple docstring'''





				__lowercase			  =							self.convert_ids_to_tokens(lowercase_	)
				__lowercase			  =							''''''.join(lowercase_	).replace(lowercase_       ,''' '''	).strip()
				return out_string


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							int:







				'''simple docstring'''





				return self.vocab.get(lowercase_       ,self.vocab.get(self.unk_token	)	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Any:







				'''simple docstring'''





				return self.reverse_vocab.get(lowercase_       ,self.unk_token	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase=None	)							->							Tuple:







				'''simple docstring'''





				if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				__lowercase			  =							[self.cls_token_id]
				__lowercase			  =							[self.sep_token_id]
				return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase=None	)							->							Any:







				'''simple docstring'''





				if offset_mapping_a is None:
							return [(0, 0)] + offset_mapping_a + [(0, 0)]

				return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase=None       ,_lowerCamelCase=False	)							->							List[str]:







				'''simple docstring'''





				if already_has_special_tokens:
							if token_ids_a is not None:
										raise ValueError(
										    '''You should not supply a second sequence if the provided sequence of '''
										    '''ids is already formatted with special tokens for the model.'''	)
							return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

				if token_ids_a is not None:
							return [1] + ([0] * len(lowercase_	)) + [1, 1] + ([0] * len(lowercase_	)) + [1]
				return [1] + ([0] * len(lowercase_	)) + [1]


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							List[int]:







				'''simple docstring'''





				if token_ids_a is None:
							# [CLS] X [SEP]
							return (len(lowercase_	) + 2) * [0]

				# [CLS] A [SEP] [SEP] B [SEP]
				return [0] * (len(lowercase_	) + 1) + [1] * (len(lowercase_	) + 3)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Optional[int]:







				'''simple docstring'''





				if "\u4e00" <= char <= "\u9fff":
							return True
				return False


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							str:







				'''simple docstring'''





				if ("a" <= char <= "z") or ("A" <= char <= "Z"):
							return True
				return False


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							List[Any]:







				'''simple docstring'''





				if char in ",;:.?!~，；：。？！《》【】":
							return True
				return False


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Any:







				'''simple docstring'''





				if char == " " or char == "\t" or char == "\n" or char == "\r":
							return True
				if len(lowercase_	) == 1:
							__lowercase			  =							unicodedata.category(lowercase_	)
							if cat == "Zs":
										return True
				return False


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Any:







				'''simple docstring'''





				__lowercase			  =							{}
				with io.open(lowercase_       ,'''r'''       ,encoding='''utf-8'''	) as f:
							for index, line in enumerate(lowercase_	):
										__lowercase			  =							line.rstrip('''\n'''	)
										__lowercase			  =							int(lowercase_	)

				return token_to_idx


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							Tuple[str]:







				'''simple docstring'''





				__lowercase			  =							0
				if os.path.isdir(lowercase_	):
							__lowercase			  =							os.path.join(
							    lowercase_       ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)
				else:
							__lowercase			  =							(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
				with open(lowercase_       ,'''w'''       ,encoding='''utf-8'''	) as writer:
							for token, token_index in sorted(self.vocab.items()       ,key=lambda _lowerCamelCase	: kv[1]	):
										if index != token_index:
													logger.warning(
													    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
													    ''' Please check that the vocabulary is not corrupted!'''	)
													__lowercase			  =							token_index
										writer.write(token + '''\n'''	)
										index += 1

				__lowercase			  =							os.path.join(lowercase_       ,'''sentencepiece.bpe.model'''	)
				with open(lowercase_       ,'''wb'''	) as fi:
							__lowercase			  =							self.sp_model.serialized_model_proto()
							fi.write(lowercase_	)

				return (vocab_file,)












'''simple docstring'''



# NOTE: This file is deprecated and will be removed in a future version.
# It only exists so that temporarely `from diffusers.pipelines import DiffusionPipeline` works

from ...utils import deprecate
from ..controlnet.pipeline_flax_controlnet import FlaxStableDiffusionControlNetPipeline  # noqa: F401


deprecate(
    '''stable diffusion controlnet''',
    '''0.22.0''',
    '''Importing `FlaxStableDiffusionControlNetPipeline` from diffusers.pipelines.stable_diffusion.flax_pipeline_stable_diffusion_controlnet is deprecated. Please import `from diffusers import FlaxStableDiffusionControlNetPipeline` instead.''',
    standard_warn=False,
    stacklevel=3,
)





