


'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import XGLMConfig, XGLMTokenizer, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers.models.xglm.modeling_tf_xglm import (
		    TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFXGLMForCausalLM,
		    TFXGLMModel,
		)
@require_tf
class        lowercase     :

		"""simple docstring"""

		_a   			=    XGLMConfig
		_a   			=    {}
		_a   			=    """gelu"""




		def __init__(		self   ,       UpperCamelCase_   ,       UpperCamelCase_=14   ,       UpperCamelCase_=7   ,       UpperCamelCase_=True   ,       UpperCamelCase_=True   ,       UpperCamelCase_=True   ,       UpperCamelCase_=99   ,       UpperCamelCase_=32   ,       UpperCamelCase_=2   ,       UpperCamelCase_=4   ,       UpperCamelCase_=37   ,       UpperCamelCase_="gelu"   ,       UpperCamelCase_=0.1   ,       UpperCamelCase_=0.1   ,       UpperCamelCase_=512   ,       UpperCamelCase_=0.02   ,       ):

							'''simple docstring'''


							UpperCamelCase__       :Optional[int]							 =			parent
							UpperCamelCase__       :Union[str, Any]							 =			batch_size
							UpperCamelCase__       :Tuple							 =			seq_length
							UpperCamelCase__       :Optional[Any]							 =			is_training
							UpperCamelCase__       :str							 =			use_input_mask
							UpperCamelCase__       :Any							 =			use_labels
							UpperCamelCase__       :List[Any]							 =			vocab_size
							UpperCamelCase__       :Optional[int]							 =			d_model
							UpperCamelCase__       :Union[str, Any]							 =			num_hidden_layers
							UpperCamelCase__       :str							 =			num_attention_heads
							UpperCamelCase__       :int							 =			ffn_dim
							UpperCamelCase__       :Optional[Any]							 =			activation_function
							UpperCamelCase__       :Optional[int]							 =			activation_dropout
							UpperCamelCase__       :Any							 =			attention_dropout
							UpperCamelCase__       :Dict							 =			max_position_embeddings
							UpperCamelCase__       :Union[str, Any]							 =			initializer_range
							UpperCamelCase__       :List[Any]							 =			None
							UpperCamelCase__       :Optional[int]							 =			0
							UpperCamelCase__       :Optional[int]							 =			2
							UpperCamelCase__       :Any							 =			1




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return XGLMConfig.from_pretrained('''facebook/xglm-564M'''							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[int]							 =			tf.clip_by_value(
							    ids_tensor([self.batch_size, self.seq_length]   ,       self.vocab_size							)   ,       clip_value_min=0   ,       clip_value_max=3							)

							UpperCamelCase__       :Any							 =			None
							if self.use_input_mask:
												UpperCamelCase__       :Dict							 =			random_attention_mask([self.batch_size, self.seq_length]							)

							UpperCamelCase__       :int							 =			self.get_config()

							UpperCamelCase__       :str							 =			floats_tensor([self.num_hidden_layers, self.num_attention_heads]   ,       2							)

							return (
							    config,
							    input_ids,
							    input_mask,
							    head_mask,
							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return XGLMConfig(
							    vocab_size=self.vocab_size   ,       d_model=self.hidden_size   ,       num_layers=self.num_hidden_layers   ,       attention_heads=self.num_attention_heads   ,       ffn_dim=self.ffn_dim   ,       activation_function=self.activation_function   ,       activation_dropout=self.activation_dropout   ,       attention_dropout=self.attention_dropout   ,       max_position_embeddings=self.max_position_embeddings   ,       initializer_range=self.initializer_range   ,       use_cache=__lowerCamelCase   ,       bos_token_id=self.bos_token_id   ,       eos_token_id=self.eos_token_id   ,       pad_token_id=self.pad_token_id   ,       return_dict=__lowerCamelCase   ,       )




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Any							 =			self.prepare_config_and_inputs()

							(
							    UpperCamelCase__				
							)       :Optional[Any]							 =			config_and_inputs

							UpperCamelCase__       :Union[str, Any]							 =			{
							    """input_ids""": input_ids,
							    """head_mask""": head_mask,
							}

							return config, inputs_dict



@require_tf
class        lowercase     (							_a     ,		_a     ,		unittest.TestCase ):

		"""simple docstring"""

		_a   			=    (TFXGLMModel, TFXGLMForCausalLM) if is_tf_available() else ()
		_a   			=    (TFXGLMForCausalLM,) if is_tf_available() else ()
		_a   			=    (
		    {"""feature-extraction""": TFXGLMModel, """text-generation""": TFXGLMForCausalLM} if is_tf_available() else {}
		)
		_a   			=    False
		_a   			=    False
		_a   			=    False




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Tuple							 =			TFXGLMModelTester(self							)
							UpperCamelCase__       :Tuple							 =			ConfigTester(self   ,       config_class=__lowerCamelCase   ,       n_embd=37							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							self.config_tester.run_common_tests()




		@slow
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							for model_name in TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												UpperCamelCase__       :Optional[int]							 =			TFXGLMModel.from_pretrained(__lowerCamelCase							)
												self.assertIsNotNone(__lowerCamelCase							)




		@unittest.skip(reason='''Currently, model embeddings are going to undergo a major refactor.'''							)
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							super().test_resize_token_embeddings()



@require_tf
class        lowercase     (							unittest.TestCase ):

		"""simple docstring"""

		@slow
		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_=True							):

							'''simple docstring'''


							UpperCamelCase__       :Dict							 =			TFXGLMForCausalLM.from_pretrained('''facebook/xglm-564M'''							)
							UpperCamelCase__       :str							 =			tf.convert_to_tensor([[2, 268, 9865]]   ,       dtype=tf.intaa							)  # The dog
							# </s> The dog is a very friendly dog. He is very affectionate and loves to play with other
							# fmt: off
							UpperCamelCase__       :Optional[Any]							 =			[2, 268, 9865, 67, 11, 1988, 57252, 9865, 5, 984, 67, 1988, 213838, 1658, 53, 70446, 33, 6657, 278, 1581]
							# fmt: on
							UpperCamelCase__       :str							 =			model.generate(__lowerCamelCase   ,       do_sample=__lowerCamelCase   ,       num_beams=1							)
							if verify_outputs:
												self.assertListEqual(output_ids[0].numpy().tolist()   ,       __lowerCamelCase							)




		@slow
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Tuple							 =			XGLMTokenizer.from_pretrained('''facebook/xglm-564M'''							)
							UpperCamelCase__       :Optional[int]							 =			TFXGLMForCausalLM.from_pretrained('''facebook/xglm-564M'''							)

							tf.random.set_seed(0							)
							UpperCamelCase__       :List[str]							 =			tokenizer('''Today is a nice day and'''   ,       return_tensors='''tf'''							)
							UpperCamelCase__       :Optional[Any]							 =			tokenized.input_ids
							# forces the generation to happen on CPU, to avoid GPU-related quirks (and assure same output regardless of the available devices)
							with tf.device(''':/CPU:0'''							):
												UpperCamelCase__       :Any							 =			model.generate(__lowerCamelCase   ,       do_sample=__lowerCamelCase   ,       seed=[7, 0]							)
							UpperCamelCase__       :Union[str, Any]							 =			tokenizer.decode(output_ids[0]   ,       skip_special_tokens=__lowerCamelCase							)

							UpperCamelCase__       :Optional[int]							 =			(
							    """Today is a nice day and warm evening here over Southern Alberta!! Today when they closed schools due"""
							)
							self.assertEqual(__lowerCamelCase   ,       __lowerCamelCase							)




		@slow
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Any							 =			TFXGLMForCausalLM.from_pretrained('''facebook/xglm-564M'''							)
							UpperCamelCase__       :List[str]							 =			XGLMTokenizer.from_pretrained('''facebook/xglm-564M'''							)

							UpperCamelCase__       :Tuple							 =			"""left"""

							# use different length sentences to test batching
							UpperCamelCase__       :List[str]							 =			[
							    """This is an extremelly long sentence that only exists to test the ability of the model to cope with """
							    """left-padding, such as in batched generation. The output for the sequence below should be the same """
							    """regardless of whether left padding is applied or not. When""",
							    """Hello, my dog is a little""",
							]

							UpperCamelCase__       :str							 =			tokenizer(__lowerCamelCase   ,       return_tensors='''tf'''   ,       padding=__lowerCamelCase							)
							UpperCamelCase__       :Optional[int]							 =			inputs["""input_ids"""]

							UpperCamelCase__       :Union[str, Any]							 =			model.generate(input_ids=__lowerCamelCase   ,       attention_mask=inputs['''attention_mask''']   ,       max_new_tokens=12							)

							UpperCamelCase__       :Optional[int]							 =			tokenizer(sentences[0]   ,       return_tensors='''tf'''							).input_ids
							UpperCamelCase__       :Dict							 =			model.generate(input_ids=__lowerCamelCase   ,       max_new_tokens=12							)

							UpperCamelCase__       :List[str]							 =			tokenizer(sentences[1]   ,       return_tensors='''tf'''							).input_ids
							UpperCamelCase__       :int							 =			model.generate(input_ids=__lowerCamelCase   ,       max_new_tokens=12							)

							UpperCamelCase__       :Any							 =			tokenizer.batch_decode(__lowerCamelCase   ,       skip_special_tokens=__lowerCamelCase							)
							UpperCamelCase__       :Any							 =			tokenizer.decode(output_non_padded[0]   ,       skip_special_tokens=__lowerCamelCase							)
							UpperCamelCase__       :List[str]							 =			tokenizer.decode(output_padded[0]   ,       skip_special_tokens=__lowerCamelCase							)

							UpperCamelCase__       :str							 =			[
							    """This is an extremelly long sentence that only exists to test the ability of the model to cope with """
							    """left-padding, such as in batched generation. The output for the sequence below should be the same """
							    """regardless of whether left padding is applied or not. When left padding is applied, the sequence will be """
							    """a single""",
							    """Hello, my dog is a little bit of a shy one, but he is very friendly""",
							]
							self.assertListEqual(__lowerCamelCase   ,       __lowerCamelCase							)
							self.assertListEqual(__lowerCamelCase   ,       [non_padded_sentence, padded_sentence]							)



'''simple docstring'''
import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
		import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case											=					logging.get_logger(__name__)

__snake_case											=					{'''vocab_file''': '''vocab.txt'''}

__snake_case											=					{
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

__snake_case											=					{
    '''openbmb/cpm-ant-10b''': 1024,
}


def       a				(      __a       )  ->     Tuple:



					'''simple docstring'''






					UpperCamelCase__       :List[str]							 =			collections.OrderedDict()
					with open(__a			, '''r'''			, encoding='''utf-8'''       ) as reader:
										UpperCamelCase__       :Dict							 =			reader.readlines()
					for index, token in enumerate(__a       ):
										UpperCamelCase__       :str							 =			token.rstrip('''\n'''       )
										UpperCamelCase__       :Optional[int]							 =			index
					return vocab
class        lowercase     (							A__ ):

		"""simple docstring"""

		def __init__(		self   ,       UpperCamelCase_   ,       UpperCamelCase_="<unk>"   ,       UpperCamelCase_=200							):

							'''simple docstring'''


							UpperCamelCase__       :Tuple							 =			vocab
							UpperCamelCase__       :List[str]							 =			unk_token
							UpperCamelCase__       :Tuple							 =			max_input_chars_per_word




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[int]							 =			list(UpperCamelCase_							)
							if len(UpperCamelCase_							) > self.max_input_chars_per_word:
												return [self.unk_token]

							UpperCamelCase__       :List[Any]							 =			0
							UpperCamelCase__       :str							 =			[]
							while start < len(UpperCamelCase_							):
												UpperCamelCase__       :int							 =			len(UpperCamelCase_							)
												UpperCamelCase__       :List[Any]							 =			None
												while start < end:
																	UpperCamelCase__       :int							 =			''''''.join(chars[start:end]							)
																	if substr in self.vocab:
																						UpperCamelCase__       :List[Any]							 =			substr
																						break
																	end -= 1
												if cur_substr is None:
																	sub_tokens.append(self.unk_token							)
																	start += 1
												else:
																	sub_tokens.append(UpperCamelCase_							)
																	UpperCamelCase__       :Any							 =			end

							return sub_tokens



class        lowercase     (							A__ ):

		"""simple docstring"""

		_a   			=    VOCAB_FILES_NAMES
		_a   			=    PRETRAINED_VOCAB_FILES_MAP
		_a   			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a   			=    ['input_ids', 'attention_mask']
		_a   			=    False




		def __init__(		self   ,       UpperCamelCase_   ,       UpperCamelCase_="<d>"   ,       UpperCamelCase_="</d>"   ,       UpperCamelCase_="<s>"   ,       UpperCamelCase_="</s>"   ,       UpperCamelCase_="<pad>"   ,       UpperCamelCase_="<unk>"   ,       UpperCamelCase_="</n>"   ,       UpperCamelCase_="</_>"   ,       UpperCamelCase_="left"   ,       **UpperCamelCase_   ,       ):

							'''simple docstring'''


							requires_backends(self   ,       ['''jieba''']							)
							super().__init__(
							    bod_token=UpperCamelCase_   ,       eod_token=UpperCamelCase_   ,       bos_token=UpperCamelCase_   ,       eos_token=UpperCamelCase_   ,       pad_token=UpperCamelCase_   ,       unk_token=UpperCamelCase_   ,       line_token=UpperCamelCase_   ,       space_token=UpperCamelCase_   ,       padding_side=UpperCamelCase_   ,       **UpperCamelCase_   ,       )
							UpperCamelCase__       :Tuple							 =			bod_token
							UpperCamelCase__       :Dict							 =			eod_token
							UpperCamelCase__       :Optional[int]							 =			load_vocab(UpperCamelCase_							)
							UpperCamelCase__       :Tuple							 =			self.encoder[space_token]
							UpperCamelCase__       :List[Any]							 =			self.encoder[line_token]

							del self.encoder[space_token]
							del self.encoder[line_token]

							UpperCamelCase__       :Union[str, Any]							 =			collections.OrderedDict(sorted(self.encoder.items()   ,       key=lambda UpperCamelCase_							: x[1]							)							)
							UpperCamelCase__       :Union[str, Any]							 =			{v: k for k, v in self.encoder.items()}

							UpperCamelCase__       :List[Any]							 =			WordpieceTokenizer(vocab=self.encoder   ,       unk_token=self.unk_token							)




		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return self.encoder[self.bod_token]




		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return self.encoder[self.eod_token]




		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return self.encoder["\n"]




		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return len(self.encoder							)




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return dict(self.encoder   ,       **self.added_tokens_encoder							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			[]
							for x in jieba.cut(UpperCamelCase_   ,       cut_all=UpperCamelCase_							):
												output_tokens.extend(self.wordpiece_tokenizer.tokenize(UpperCamelCase_							)							)
							return output_tokens




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       **UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			[i for i in token_ids if i >= 0]
							UpperCamelCase__       :Optional[int]							 =			[
							    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
							]
							return super()._decode(UpperCamelCase_   ,       **UpperCamelCase_							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return token in self.encoder




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return "".join(UpperCamelCase_							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return self.encoder.get(UpperCamelCase_   ,       self.encoder.get(self.unk_token							)							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return self.decoder.get(UpperCamelCase_   ,       self.unk_token							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None							):

							'''simple docstring'''


							if os.path.isdir(UpperCamelCase_							):
												UpperCamelCase__       :int							 =			os.path.join(
												    UpperCamelCase_   ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
							else:
												UpperCamelCase__       :str							 =			(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
							UpperCamelCase__       :Any							 =			0
							if " " in self.encoder:
												UpperCamelCase__       :Dict							 =			self.encoder[''' ''']
												del self.encoder[" "]
							if "\n" in self.encoder:
												UpperCamelCase__       :List[str]							 =			self.encoder['''\n''']
												del self.encoder["\n"]
							UpperCamelCase__       :List[str]							 =			collections.OrderedDict(sorted(self.encoder.items()   ,       key=lambda UpperCamelCase_							: x[1]							)							)
							with open(UpperCamelCase_   ,       '''w'''   ,       encoding='''utf-8'''							) as writer:
												for token, token_index in self.encoder.items():
																	if index != token_index:
																						logger.warning(
																						    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																						    ''' Please check that the vocabulary is not corrupted!'''							)
																						UpperCamelCase__       :Any							 =			token_index
																	writer.write(token + '''\n'''							)
																	index += 1
							return (vocab_file,)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None							):

							'''simple docstring'''


							if token_ids_a is None:
												return [self.bos_token_id] + token_ids_a
							return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None   ,       UpperCamelCase_ = False							):

							'''simple docstring'''


							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=UpperCamelCase_   ,       token_ids_a=UpperCamelCase_   ,       already_has_special_tokens=UpperCamelCase_							)

							if token_ids_a is not None:
												return [1] + ([0] * len(UpperCamelCase_							)) + [1] + ([0] * len(UpperCamelCase_							))
							return [1] + ([0] * len(UpperCamelCase_							))