



'''simple docstring'''

import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase     		=   logging.get_logger(__name__)

UpperCAmelCase     		=   '▁'

UpperCAmelCase     		=   {'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

UpperCAmelCase     		=   {
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

UpperCAmelCase     		=   {
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

UpperCAmelCase     		=   {
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

UpperCAmelCase     		=   {
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}

class __snake_case(				_lowerCAmelCase ):


						'''simple docstring'''



						UpperCAmelCase							:   List[str]									=		["input_ids"]

						UpperCAmelCase							:   Any									=		VOCAB_FILES_NAMES
						UpperCAmelCase							:   List[Any]									=		PRETRAINED_INIT_CONFIGURATION
						UpperCAmelCase							:   Union[str, Any]									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase							:   Optional[Any]									=		PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase							:   Tuple									=		RESOURCE_FILES_NAMES


						def __init__( self     ,	A_     ,	A_=None     ,	A_=False     ,	A_="utf8"     ,	A_="[UNK]"     ,	A_="[SEP]"     ,	A_="[PAD]"     ,	A_="[CLS]"     ,	A_="[MASK]"     ,	A_ = None     ,	**A_     ,	)     ->     None:
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.

							lowerCAmelCase   =     {} if sp_model_kwargs is None else sp_model_kwargs
							super().__init__(
							    do_lower_case=A_     ,	unk_token=A_     ,	sep_token=A_     ,	pad_token=A_     ,	cls_token=A_     ,	mask_token=A_     ,	vocab_file=A_     ,	encoding=A_     ,	sp_model_kwargs=self.sp_model_kwargs     ,	**A_     ,	)
							lowerCAmelCase   =     do_lower_case
							lowerCAmelCase   =     sentencepiece_model_ckpt
							lowerCAmelCase   =     spm.SentencePieceProcessor(**self.sp_model_kwargs				)
							self.sp_model.Load(A_				)

							# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
							if vocab_file is not None:
								lowerCAmelCase   =     self.load_vocab(filepath=A_				)
							else:
								lowerCAmelCase   =     {self.sp_model.id_to_piece(A_				): id for id in range(self.sp_model.get_piece_size()				)}
							lowerCAmelCase   =     {v: k for k, v in self.vocab.items()}


						def     __snake_case   ( self     ,	A_				)     ->     int:
							if text is None:
								return None

							lowerCAmelCase   =     self.tokenize(A_				)
							lowerCAmelCase,       lowerCAmelCase   =     """""", []

							for i, ch in enumerate(A_				):
								if ch in self.SP_CHAR_MAPPING:
									lowerCAmelCase   =     self.SP_CHAR_MAPPING.get(A_				)
								else:
									lowerCAmelCase   =     unicodedata.normalize("""NFKC"""     ,	A_				)
								if self.is_whitespace(A_				):
									continue
								normalized_text += ch
								char_mapping.extend([i] * len(A_				)				)

							lowerCAmelCase,       lowerCAmelCase,       lowerCAmelCase   =     normalized_text, [], 0

							if self.do_lower_case:
								lowerCAmelCase   =     text.lower()

							for token in split_tokens:
								if token[:1] == "▁":
									lowerCAmelCase   =     token[1:]
								lowerCAmelCase   =     text[offset:].index(A_				) + offset
								lowerCAmelCase   =     start + len(A_				)

								token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)				)
								lowerCAmelCase   =     end
							return token_mapping


						@property
						def     __snake_case   ( self				)     ->     List[str]:
							return len(self.vocab				)


						def     __snake_case   ( self				)     ->     int:
							return dict(self.vocab     ,	**self.added_tokens_encoder				)


						def __getstate__( self				)     ->     int:
							lowerCAmelCase   =     self.__dict__.copy()
							lowerCAmelCase   =     None
							return state


						def __setstate__( self     ,	A_				)     ->     Tuple:
							lowerCAmelCase   =     d

							# for backward compatibility
							if not hasattr(self     ,	"""sp_model_kwargs"""				):
								lowerCAmelCase   =     {}

							lowerCAmelCase   =     spm.SentencePieceProcessor(**self.sp_model_kwargs				)
							self.sp_model.Load(self.sentencepiece_model_ckpt				)


						def     __snake_case   ( self     ,	A_				)     ->     Any:
							return "".join((self.SP_CHAR_MAPPING.get(A_     ,	A_				) for c in text)				)


						def     __snake_case   ( self     ,	A_     ,	A_=False     ,	A_=64     ,	A_=0.1				)     ->     Optional[int]:

							if self.sp_model_kwargs.get("""enable_sampling"""				) is True:
								lowerCAmelCase   =     True
							if self.sp_model_kwargs.get("""alpha"""				) is not None:
								lowerCAmelCase   =     self.sp_model_kwargs.get("""alpha"""				)
							if self.sp_model_kwargs.get("""nbest_size"""				) is not None:
								lowerCAmelCase   =     self.sp_model_kwargs.get("""nbest_size"""				)

							if not enable_sampling:
								lowerCAmelCase   =     self.sp_model.EncodeAsPieces(A_				)
							else:
								lowerCAmelCase   =     self.sp_model.SampleEncodeAsPieces(A_     ,	A_     ,	A_				)
							lowerCAmelCase   =     []
							for pi, piece in enumerate(A_				):
								if piece == SPIECE_UNDERLINE:
									if not pieces[pi + 1].startswith(A_				) and pi != 0:
										new_pieces.append(A_				)
										continue
									else:
										continue
								lowerCAmelCase   =     0
								for i, chunk in enumerate(A_				):
									if chunk == SPIECE_UNDERLINE:
										continue
									if self.is_ch_char(A_				) or self.is_punct(A_				):
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]				)
										new_pieces.append(A_				)
										lowerCAmelCase   =     i + 1
									elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]				)
										lowerCAmelCase   =     i
									elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
											new_pieces.append(piece[lst_i:i]				)
										lowerCAmelCase   =     i
								if len(A_				) > lst_i:
									new_pieces.append(piece[lst_i:]				)
							return new_pieces


						def     __snake_case   ( self     ,	A_				)     ->     Tuple:
							lowerCAmelCase   =     """""".join(A_				).replace(A_     ,	""" """				).strip()
							return out_string


						def     __snake_case   ( self     ,	A_				)     ->     Optional[int]:
							lowerCAmelCase   =     self.convert_ids_to_tokens(A_				)
							lowerCAmelCase   =     """""".join(A_				).replace(A_     ,	""" """				).strip()
							return out_string


						def     __snake_case   ( self     ,	A_				)     ->     List[Any]:
							return self.vocab.get(A_     ,	self.vocab.get(self.unk_token				)				)


						def     __snake_case   ( self     ,	A_				)     ->     int:
							return self.reverse_vocab.get(A_     ,	self.unk_token				)


						def     __snake_case   ( self     ,	A_     ,	A_=None				)     ->     int:
							if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							lowerCAmelCase   =     [self.cls_token_id]
							lowerCAmelCase   =     [self.sep_token_id]
							return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


						def     __snake_case   ( self     ,	A_     ,	A_=None				)     ->     int:
							if offset_mapping_a is None:
								return [(0, 0)] + offset_mapping_a + [(0, 0)]

							return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


						def     __snake_case   ( self     ,	A_     ,	A_=None     ,	A_=False				)     ->     List[str]:

							if already_has_special_tokens:
								if token_ids_a is not None:
									raise ValueError(
									    """You should not supply a second sequence if the provided sequence of """
									    """ids is already formatted with special tokens for the model."""				)
								return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

							if token_ids_a is not None:
								return [1] + ([0] * len(A_				)) + [1, 1] + ([0] * len(A_				)) + [1]
							return [1] + ([0] * len(A_				)) + [1]


						def     __snake_case   ( self     ,	A_     ,	A_ = None				)     ->     List[int]:
							# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
							if token_ids_a is None:
								# [CLS] X [SEP]
								return (len(A_				) + 2) * [0]

							# [CLS] A [SEP] [SEP] B [SEP]
							return [0] * (len(A_				) + 1) + [1] * (len(A_				) + 3)


						def     __snake_case   ( self     ,	A_				)     ->     Tuple:
							if "\u4e00" <= char <= "\u9fff":
								return True
							return False


						def     __snake_case   ( self     ,	A_				)     ->     List[Any]:
							if ("a" <= char <= "z") or ("A" <= char <= "Z"):
								return True
							return False


						def     __snake_case   ( self     ,	A_				)     ->     str:
							if char in ",;:.?!~，；：。？！《》【】":
								return True
							return False


						def     __snake_case   ( self     ,	A_				)     ->     List[str]:
							if char == " " or char == "\t" or char == "\n" or char == "\r":
								return True
							if len(A_				) == 1:
								lowerCAmelCase   =     unicodedata.category(A_				)
								if cat == "Zs":
									return True
							return False


						def     __snake_case   ( self     ,	A_				)     ->     List[str]:
							lowerCAmelCase   =     {}
							with io.open(A_     ,	"""r"""     ,	encoding="""utf-8"""				) as f:
								for index, line in enumerate(A_				):
									lowerCAmelCase   =     line.rstrip("""\n"""				)
									lowerCAmelCase   =     int(A_				)

							return token_to_idx




						def     __snake_case   ( self     ,	A_     ,	A_ = None				)     ->     Tuple[str]:
							lowerCAmelCase   =     0
							if os.path.isdir(A_				):
								lowerCAmelCase   =     os.path.join(
								    A_     ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
							else:
								lowerCAmelCase   =     (filename_prefix + """-""" if filename_prefix else """""") + save_directory
							with open(A_     ,	"""w"""     ,	encoding="""utf-8"""				) as writer:
								for token, token_index in sorted(self.vocab.items()     ,	key=lambda A_				: kv[1]				):
									if index != token_index:
										logger.warning(
										    f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
										    """ Please check that the vocabulary is not corrupted!"""				)
										lowerCAmelCase   =     token_index
									writer.write(token + """\n"""				)
									index += 1

							lowerCAmelCase   =     os.path.join(A_     ,	"""sentencepiece.bpe.model"""				)
							with open(A_     ,	"""wb"""				) as fi:
								lowerCAmelCase   =     self.sp_model.serialized_model_proto()
								fi.write(A_				)

							return (vocab_file,)




'''simple docstring'''

import tensorflow as tf

from ...tf_utils import shape_list

class __snake_case(				tf.keras.layers.Layer ):


						'''simple docstring'''



						def __init__( self     ,	A_     ,	A_     ,	A_     ,	A_     ,	A_=1     ,	A_=False     ,	**A_				)     ->     Optional[int]:
							super().__init__(**A_				)

							lowerCAmelCase   =     vocab_size
							lowerCAmelCase   =     d_embed
							lowerCAmelCase   =     d_proj

							lowerCAmelCase   =     cutoffs + [vocab_size]
							lowerCAmelCase   =     [0] + self.cutoffs
							lowerCAmelCase   =     div_val

							lowerCAmelCase   =     self.cutoffs[0]
							lowerCAmelCase   =     len(self.cutoffs				) - 1
							lowerCAmelCase   =     self.shortlist_size + self.n_clusters
							lowerCAmelCase   =     keep_order

							lowerCAmelCase   =     []
							lowerCAmelCase   =     []


						def     __snake_case   ( self     ,	A_				)     ->     int:
							if self.n_clusters > 0:
								lowerCAmelCase   =     self.add_weight(
								    shape=(self.n_clusters, self.d_embed)     ,	initializer="""zeros"""     ,	trainable=A_     ,	name="""cluster_weight"""				)
								lowerCAmelCase   =     self.add_weight(
								    shape=(self.n_clusters,)     ,	initializer="""zeros"""     ,	trainable=A_     ,	name="""cluster_bias"""				)

							if self.div_val == 1:
								for i in range(len(self.cutoffs				)				):
									if self.d_proj != self.d_embed:
										lowerCAmelCase   =     self.add_weight(
										    shape=(self.d_embed, self.d_proj)     ,	initializer="""zeros"""     ,	trainable=A_     ,	name=f'out_projs_._{i}'     ,	)
										self.out_projs.append(A_				)
									else:
										self.out_projs.append(A_				)
									lowerCAmelCase   =     self.add_weight(
									    shape=(self.vocab_size, self.d_embed)     ,	initializer="""zeros"""     ,	trainable=A_     ,	name=f'out_layers_._{i}_._weight'     ,	)
									lowerCAmelCase   =     self.add_weight(
									    shape=(self.vocab_size,)     ,	initializer="""zeros"""     ,	trainable=A_     ,	name=f'out_layers_._{i}_._bias'     ,	)
									self.out_layers.append((weight, bias)				)
							else:
								for i in range(len(self.cutoffs				)				):
									lowerCAmelCase,       lowerCAmelCase   =     self.cutoff_ends[i], self.cutoff_ends[i + 1]
									lowerCAmelCase   =     self.d_embed // (self.div_val**i)

									lowerCAmelCase   =     self.add_weight(
									    shape=(d_emb_i, self.d_proj)     ,	initializer="""zeros"""     ,	trainable=A_     ,	name=f'out_projs_._{i}'				)
									self.out_projs.append(A_				)
									lowerCAmelCase   =     self.add_weight(
									    shape=(r_idx - l_idx, d_emb_i)     ,	initializer="""zeros"""     ,	trainable=A_     ,	name=f'out_layers_._{i}_._weight'     ,	)
									lowerCAmelCase   =     self.add_weight(
									    shape=(r_idx - l_idx,)     ,	initializer="""zeros"""     ,	trainable=A_     ,	name=f'out_layers_._{i}_._bias'     ,	)
									self.out_layers.append((weight, bias)				)
							super().build(A_				)


						@staticmethod
						def     __snake_case   ( A_     ,	A_     ,	A_     ,	A_=None				)     ->     List[Any]:
							lowerCAmelCase   =     x
							if proj is not None:
								lowerCAmelCase   =     tf.einsum("""ibd,ed->ibe"""     ,	A_     ,	A_				)
							return tf.einsum("""ibd,nd->ibn"""     ,	A_     ,	A_				) + b


						@staticmethod
						def     __snake_case   ( A_     ,	A_				)     ->     Dict:
							lowerCAmelCase   =     shape_list(A_				)
							lowerCAmelCase   =     tf.range(lp_size[0]     ,	dtype=target.dtype				)
							lowerCAmelCase   =     tf.stack([r, target]     ,	1				)
							return tf.gather_nd(A_     ,	A_				)




						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_=True     ,	A_=False				)     ->     Tuple:
							lowerCAmelCase   =     0
							if self.n_clusters == 0:
								lowerCAmelCase   =     self._logit(A_     ,	self.out_layers[0][0]     ,	self.out_layers[0][1]     ,	self.out_projs[0]				)
								if target is not None:
									lowerCAmelCase   =     tf.nn.sparse_softmax_cross_entropy_with_logits(labels=A_     ,	logits=A_				)
								lowerCAmelCase   =     tf.nn.log_softmax(A_     ,	axis=-1				)
							else:
								lowerCAmelCase   =     shape_list(A_				)
								lowerCAmelCase   =     []
								lowerCAmelCase   =     tf.zeros(hidden_sizes[:2]				)
								for i in range(len(self.cutoffs				)				):
									lowerCAmelCase,       lowerCAmelCase   =     self.cutoff_ends[i], self.cutoff_ends[i + 1]
									if target is not None:
										lowerCAmelCase   =     (target >= l_idx) & (target < r_idx)
										lowerCAmelCase   =     tf.where(A_				)
										lowerCAmelCase   =     tf.boolean_mask(A_     ,	A_				) - l_idx

									if self.div_val == 1:
										lowerCAmelCase   =     self.out_layers[0][0][l_idx:r_idx]
										lowerCAmelCase   =     self.out_layers[0][1][l_idx:r_idx]
									else:
										lowerCAmelCase   =     self.out_layers[i][0]
										lowerCAmelCase   =     self.out_layers[i][1]

									if i == 0:
										lowerCAmelCase   =     tf.concat([cur_W, self.cluster_weight]     ,	0				)
										lowerCAmelCase   =     tf.concat([cur_b, self.cluster_bias]     ,	0				)

										lowerCAmelCase   =     self._logit(A_     ,	A_     ,	A_     ,	self.out_projs[0]				)
										lowerCAmelCase   =     tf.nn.log_softmax(A_				)
										out.append(head_logprob[..., : self.cutoffs[0]]				)
										if target is not None:
											lowerCAmelCase   =     tf.boolean_mask(A_     ,	A_				)
											lowerCAmelCase   =     self._gather_logprob(A_     ,	A_				)
									else:
										lowerCAmelCase   =     self._logit(A_     ,	A_     ,	A_     ,	self.out_projs[i]				)
										lowerCAmelCase   =     tf.nn.log_softmax(A_				)
										lowerCAmelCase   =     self.cutoffs[0] + i - 1  # No probability for the head cluster
										lowerCAmelCase   =     head_logprob[..., cluster_prob_idx, None] + tail_logprob
										out.append(A_				)
										if target is not None:
											lowerCAmelCase   =     tf.boolean_mask(A_     ,	A_				)
											lowerCAmelCase   =     tf.boolean_mask(A_     ,	A_				)
											lowerCAmelCase   =     self._gather_logprob(A_     ,	A_				)
											cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
									if target is not None:
										loss += tf.scatter_nd(A_     ,	-cur_logprob     ,	shape_list(A_				)				)
								lowerCAmelCase   =     tf.concat(A_     ,	axis=-1				)

							if target is not None:
								if return_mean:
									lowerCAmelCase   =     tf.reduce_mean(A_				)
								# Add the training-time loss value to the layer using `self.add_loss()`.
								self.add_loss(A_				)

								# Log the loss as a metric (we could log arbitrary metrics,
								# including different metrics for training and inference.
								self.add_metric(A_     ,	name=self.name     ,	aggregation="""mean""" if return_mean else """"""				)

							return out