



"""simple docstring"""




import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_							:      int      =  get_tests_dir("fixtures/test_sentencepiece.model")






@require_sentencepiece
@require_tokenizers
class      UpperCamelCase    (						__lowercase       ,    unittest.TestCase     ):
	lowerCAmelCase	:   str				=	ReformerTokenizer
	lowerCAmelCase	:   str				=	ReformerTokenizerFast
	lowerCAmelCase	:   Optional[int]				=	True
	lowerCAmelCase	:   str				=	False
	lowerCAmelCase	:   int				=	True



	def 			__A							(       self		):
							super().setUp()

							A__							=       ReformerTokenizer(_a			,	keep_accents=_a		)
							tokenizer.save_pretrained(self.tmpdirname		)



	def 			__A							(       self		):
							A__							=       "<s>"
							A__							=       1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a		)			,	_a		)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a		)			,	_a		)



	def 			__A							(       self		):
							A__							=       list(self.get_tokenizer().get_vocab().keys()		)

							self.assertEqual(vocab_keys[0]			,	"<unk>"		)
							self.assertEqual(vocab_keys[1]			,	"<s>"		)
							self.assertEqual(vocab_keys[-1]			,	"j"		)
							self.assertEqual(len(_a		)			,	1_000		)



	def 			__A							(       self		):
							self.assertEqual(self.get_tokenizer().vocab_size			,	1_000		)



	def 			__A							(       self		):
							if not self.test_rust_tokenizer:
													return

							A__							=       self.get_tokenizer()
							A__							=       self.get_rust_tokenizer()

							A__							=       "I was born in 92000, and this is falsé."

							A__							=       tokenizer.tokenize(_a		)
							A__							=       rust_tokenizer.tokenize(_a		)
							self.assertListEqual(_a			,	_a		)

							A__							=       tokenizer.encode(_a			,	add_special_tokens=_a		)
							A__							=       rust_tokenizer.encode(_a			,	add_special_tokens=_a		)
							self.assertListEqual(_a			,	_a		)

							A__							=       self.get_rust_tokenizer()
							A__							=       tokenizer.encode(_a		)
							A__							=       rust_tokenizer.encode(_a		)
							self.assertListEqual(_a			,	_a		)



	def 			__A							(       self			,	UpperCAmelCase__=15		):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
																			A__							=       self.rust_tokenizer_class.from_pretrained(_a			,	**_a		)

																			# Simple input
																			A__							=       "This is a simple input"
																			A__							=       ["This is a simple input 1", "This is a simple input 2"]
																			A__							=       ("This is a simple input", "This is a pair")
																			A__							=       [
																			    ("This is a simple input 1", "This is a simple input 2"),
																			    ("This is a simple pair 1", "This is a simple pair 2"),
																			]

																			# Simple input tests
																			self.assertRaises(_a			,	tokenizer_r.encode			,	_a			,	max_length=_a			,	padding="max_length"		)

																			# Simple input
																			self.assertRaises(_a			,	tokenizer_r.encode_plus			,	_a			,	max_length=_a			,	padding="max_length"		)

																			# Simple input
																			self.assertRaises(
																			    _a			,	tokenizer_r.batch_encode_plus			,	_a			,	max_length=_a			,	padding="max_length"			,	)

																			# Pair input
																			self.assertRaises(_a			,	tokenizer_r.encode			,	_a			,	max_length=_a			,	padding="max_length"		)

																			# Pair input
																			self.assertRaises(_a			,	tokenizer_r.encode_plus			,	_a			,	max_length=_a			,	padding="max_length"		)

																			# Pair input
																			self.assertRaises(
																			    _a			,	tokenizer_r.batch_encode_plus			,	_a			,	max_length=_a			,	padding="max_length"			,	)



	def 			__A							(       self		):
							pass



	def 			__A							(       self		):
							A__							=       ReformerTokenizer(_a			,	keep_accents=_a		)

							A__							=       tokenizer.tokenize("This is a test"		)
							self.assertListEqual(_a			,	["▁This", "▁is", "▁a", "▁t", "est"]		)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(_a		)			,	[285, 46, 10, 170, 382]			,	)

							A__							=       tokenizer.tokenize("I was born in 92000, and this is falsé."		)
							self.assertListEqual(
							    _a			,	[
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "é",
							        ".",
							    ]			,	)
							A__							=       tokenizer.convert_tokens_to_ids(_a		)
							self.assertListEqual(
							    _a			,	[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]			,	)

							A__							=       tokenizer.convert_ids_to_tokens(_a		)
							self.assertListEqual(
							    _a			,	[
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "<unk>",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "<unk>",
							        ".",
							    ]			,	)



	@cached_property
	def 			__A							(       self		):
							return ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment"		)



	@slow
	def 			__A							(       self		):
							A__							=       "Hello World!"
							A__							=       [126, 32, 262, 152, 38, 72, 287]

							self.assertListEqual(_a			,	self.big_tokenizer.encode(_a		)		)



	@slow
	def 			__A							(       self		):
							A__							=       (
							    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
							    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
							)
							A__							=       [
							    108,
							    265,
							    24,
							    111,
							    4,
							    258,
							    156,
							    35,
							    28,
							    275,
							    3,
							    259,
							    297,
							    260,
							    84,
							    4,
							    35,
							    110,
							    44,
							    8,
							    259,
							    91,
							    268,
							    21,
							    11,
							    209,
							    274,
							    109,
							    266,
							    277,
							    117,
							    86,
							    93,
							    315,
							    258,
							    278,
							    258,
							    277,
							    258,
							    0,
							    258,
							    288,
							    258,
							    319,
							    258,
							    0,
							    258,
							    0,
							    258,
							    0,
							    258,
							    0,
							    258,
							    287,
							    258,
							    315,
							    258,
							    289,
							    258,
							    278,
							    99,
							    269,
							    266,
							    262,
							    8,
							    259,
							    241,
							    4,
							    217,
							    230,
							    268,
							    266,
							    55,
							    168,
							    106,
							    75,
							    193,
							    266,
							    223,
							    27,
							    49,
							    26,
							    282,
							    25,
							    264,
							    299,
							    19,
							    26,
							    0,
							    258,
							    277,
							    117,
							    86,
							    93,
							    176,
							    183,
							    270,
							    11,
							    262,
							    42,
							    61,
							    265,
							]

							self.assertListEqual(_a			,	self.big_tokenizer.encode(_a		)		)



	@require_torch
	@slow
	def 			__A							(       self		):
							import torch

							from transformers import ReformerConfig, ReformerModel

							# Build sequence
							A__							=       list(self.big_tokenizer.get_vocab().keys()		)[:10]
							A__							=       " ".join(_a		)
							A__							=       self.big_tokenizer.encode_plus(_a			,	return_tensors="pt"		)
							A__							=       self.big_tokenizer.batch_encode_plus([sequence, sequence]			,	return_tensors="pt"		)

							A__							=       ReformerConfig()
							# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
							A__							=       encoded_sequence["input_ids"].shape
							A__							=       ReformerModel(_a		)

							# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
							assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

							with torch.no_grad():
													model(**_a		)
													model(**_a		)



	@slow
	def 			__A							(       self		):
							# fmt: off
							A__							=       {"input_ids": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							# This tokenizer does not know some characters like ")".
							# That is the reason why we use very simple texts here.
							# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
							A__							=       [
							    "This is a very simple sentence.",
							    "The quick brown fox jumps over the lazy dog.",
							]

							self.tokenizer_integration_test_util(
							    expected_encoding=_a			,	model_name="google/reformer-crime-and-punishment"			,	revision="0e6c3decb8211d49bf881013425dc8b0448b3f5a"			,	padding=_a			,	sequences=_a			,	)










from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

UpperCAmelCase_							:      List[Any]      =  [
    "python",
    "tqdm",
    "regex",
    "requests",
    "packaging",
    "filelock",
    "numpy",
    "tokenizers",
    "huggingface-hub",
    "safetensors",
    "accelerate",
    "pyyaml",
]

for pkg in pkgs_to_check_at_runtime:
						if pkg in deps:
												if pkg == "tokenizers":
																		# must be loaded here, or else tqdm check may fail
																		from .utils import is_tokenizers_available

																		if not is_tokenizers_available():
																								continue  # not required, check version only if installed
												elif pkg == "accelerate":
																		# must be loaded here, or else tqdm check may fail
																		from .utils import is_accelerate_available

																		# Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of
																		# Transformers with PyTorch
																		if not is_accelerate_available():
																								continue  # not required, check version only if installed

												require_version_core(deps[pkg])
						else:
												raise ValueError(F'''can\'t find {pkg} in {deps.keys()}, check dependency_versions_table.py''')
def  UpperCamelCase						(       _A			:				List[Any]			,							_A			:				int=None    )->       Optional[int]:





						"""simple docstring"""





						require_version(deps[pkg]			,							_A    )





