



import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_							:      str      =  "▁"

UpperCAmelCase_							:      Optional[int]      =  get_tests_dir("fixtures/test_sentencepiece.model")






@require_sentencepiece
@require_tokenizers
class      UpperCamelCase    (						_UpperCAmelCase       ,    unittest.TestCase     ):
	lowerCAmelCase	:   Any				=	BigBirdTokenizer
	lowerCAmelCase	:   Tuple				=	BigBirdTokenizerFast
	lowerCAmelCase	:   Optional[int]				=	True
	lowerCAmelCase	:   Tuple				=	True



	def 			__A							(       self		):
							super().setUp()

							A__							=       self.tokenizer_class(UpperCAmelCase__			,	keep_accents=UpperCAmelCase__		)
							tokenizer.save_pretrained(self.tmpdirname		)



	def 			__A							(       self		):
							A__							=       "<s>"
							A__							=       1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase__		)			,	UpperCAmelCase__		)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase__		)			,	UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       list(self.get_tokenizer().get_vocab().keys()		)

							self.assertEqual(vocab_keys[0]			,	"<unk>"		)
							self.assertEqual(vocab_keys[1]			,	"<s>"		)
							self.assertEqual(vocab_keys[-1]			,	"[MASK]"		)
							self.assertEqual(len(UpperCAmelCase__		)			,	1_004		)



	def 			__A							(       self		):
							self.assertEqual(self.get_tokenizer().vocab_size			,	1_000		)



	def 			__A							(       self		):
							if not self.test_rust_tokenizer:
													return

							A__							=       self.get_tokenizer()
							A__							=       self.get_rust_tokenizer()

							A__							=       "I was born in 92000, and this is falsé."

							A__							=       tokenizer.tokenize(UpperCAmelCase__		)
							A__							=       rust_tokenizer.tokenize(UpperCAmelCase__		)
							self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)

							A__							=       tokenizer.encode(UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__		)
							A__							=       rust_tokenizer.encode(UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__		)
							self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)

							A__							=       self.get_rust_tokenizer()
							A__							=       tokenizer.encode(UpperCAmelCase__		)
							A__							=       rust_tokenizer.encode(UpperCAmelCase__		)
							self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       BigBirdTokenizer(UpperCAmelCase__			,	keep_accents=UpperCAmelCase__		)

							A__							=       tokenizer.tokenize("This is a test"		)
							self.assertListEqual(UpperCAmelCase__			,	["▁This", "▁is", "▁a", "▁t", "est"]		)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(UpperCAmelCase__		)			,	[285, 46, 10, 170, 382]			,	)

							A__							=       tokenizer.tokenize("I was born in 92000, and this is falsé."		)
							self.assertListEqual(
							    UpperCAmelCase__			,	[
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "é",
							        ".",
							    ]			,	)
							A__							=       tokenizer.convert_tokens_to_ids(UpperCAmelCase__		)
							self.assertListEqual(
							    UpperCAmelCase__			,	[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]			,	)

							A__							=       tokenizer.convert_ids_to_tokens(UpperCAmelCase__		)
							self.assertListEqual(
							    UpperCAmelCase__			,	[
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "<unk>",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "<unk>",
							        ".",
							    ]			,	)



	@cached_property
	def 			__A							(       self		):
							return BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base"		)



	@slow
	def 			__A							(       self		):
							A__							=       "Hello World!"
							A__							=       [65, 18_536, 2_260, 101, 66]

							self.assertListEqual(UpperCAmelCase__			,	self.big_tokenizer.encode(UpperCAmelCase__		)		)



	@slow
	def 			__A							(       self		):
							A__							=       (
							    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
							    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
							)
							# fmt: off
							A__							=       [65, 871, 419, 358, 946, 991, 2_521, 452, 358, 1_357, 387, 7_751, 3_536, 112, 985, 456, 126, 865, 938, 5_400, 5_734, 458, 1_368, 467, 786, 2_462, 5_246, 1_159, 633, 865, 4_519, 457, 582, 852, 2_557, 427, 916, 508, 405, 34_324, 497, 391, 408, 11_342, 1_244, 385, 100, 938, 985, 456, 574, 362, 12_597, 3_200, 3_129, 1_172, 66]  # noqa: E231
							# fmt: on
							self.assertListEqual(UpperCAmelCase__			,	self.big_tokenizer.encode(UpperCAmelCase__		)		)



	@require_torch
	@slow
	def 			__A							(       self		):
							import torch

							from transformers import BigBirdConfig, BigBirdModel

							# Build sequence
							A__							=       list(self.big_tokenizer.get_vocab().keys()		)[:10]
							A__							=       " ".join(UpperCAmelCase__		)
							A__							=       self.big_tokenizer.encode_plus(UpperCAmelCase__			,	return_tensors="pt"			,	return_token_type_ids=UpperCAmelCase__		)
							A__							=       self.big_tokenizer.batch_encode_plus(
							    [sequence + " " + sequence]			,	return_tensors="pt"			,	return_token_type_ids=UpperCAmelCase__		)

							A__							=       BigBirdConfig(attention_type="original_full"		)
							A__							=       BigBirdModel(UpperCAmelCase__		)

							assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

							with torch.no_grad():
													model(**UpperCAmelCase__		)
													model(**UpperCAmelCase__		)



	@slow
	def 			__A							(       self		):
							A__							=       BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base"		)
							A__							=       tokenizer.decode(tokenizer("Paris is the [MASK]."		).input_ids		)

							self.assertTrue(decoded_text == "[CLS] Paris is the[MASK].[SEP]"		)



	@slow
	def 			__A							(       self		):
							# fmt: off
							A__							=       {"input_ids": [[65, 39_286, 458, 36_335, 2_001, 456, 13_073, 13_266, 455, 113, 7_746, 1_741, 11_157, 391, 13_073, 13_266, 455, 113, 3_967, 35_412, 113, 4_936, 109, 3_870, 2_377, 113, 30_084, 45_720, 458, 134, 17_496, 112, 503, 11_672, 113, 118, 112, 5_665, 13_347, 38_687, 112, 1_496, 31_389, 112, 3_268, 47_264, 134, 962, 112, 16_377, 8_035, 23_130, 430, 12_169, 15_518, 28_592, 458, 146, 41_697, 109, 391, 12_169, 15_518, 16_689, 458, 146, 41_358, 109, 452, 726, 4_034, 111, 763, 35_412, 5_082, 388, 1_903, 111, 9_051, 391, 2_870, 48_918, 1_900, 1_123, 550, 998, 112, 9_586, 15_985, 455, 391, 410, 22_955, 37_636, 114, 66], [65, 448, 17_496, 419, 3_663, 385, 763, 113, 27_533, 2_870, 3_283, 13_043, 1_639, 24_713, 523, 656, 24_013, 18_550, 2_521, 517, 27_014, 21_244, 420, 1_212, 1_465, 391, 927, 4_833, 388, 578, 11_786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2_169, 7_687, 21_932, 18_146, 726, 363, 17_032, 3_391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=UpperCAmelCase__			,	model_name="google/bigbird-roberta-base"			,	revision="215c99f1600e06f83acce68422f2035b2b5c3510"			,	)










def  UpperCamelCase						(       _A			:				str    )->       str:





						"""simple docstring"""





						A__							=       ""
						for ch in key:
												if ch == " " or ch not in key_no_dups and ch.isalpha():
																		key_no_dups += ch
						return key_no_dups
def  UpperCamelCase						(       _A			:				str    )->       dict[str, str]:





						"""simple docstring"""





						A__							=       [chr(i + 65    ) for i in range(26    )]
						# Remove duplicate characters from key
						A__							=       remove_duplicates(key.upper()    )
						A__							=       len(_A    )
						# First fill cipher with key characters
						A__							=       {alphabet[i]: char for i, char in enumerate(_A    )}
						# Then map remaining characters in alphabet to
						# the alphabet from the beginning
						for i in range(len(_A    )			,							26    ):
												A__							=       alphabet[i - offset]
												# Ensure we are not mapping letters to letters previously mapped
												while char in key:
																		offset -= 1
																		A__							=       alphabet[i - offset]
												A__							=       char
						return cipher_alphabet
def  UpperCamelCase						(       _A			:				str			,							_A			:				dict[str, str]    )->       str:





						"""simple docstring"""





						return "".join(cipher_map.get(_A			,							_A    ) for ch in message.upper()    )
def  UpperCamelCase						(       _A			:				str			,							_A			:				dict[str, str]    )->       str:





						"""simple docstring"""





						A__							=       {v: k for k, v in cipher_map.items()}
						return "".join(rev_cipher_map.get(_A			,							_A    ) for ch in message.upper()    )
def  UpperCamelCase						(       )->       None:





						"""simple docstring"""





						A__							=       input("Enter message to encode or decode: "    ).strip()
						A__							=       input("Enter keyword: "    ).strip()
						A__							=       input("Encipher or decipher? E/D:"    ).strip()[0].lower()
						try:
												A__							=       {"e": encipher, "d": decipher}[option]
						except KeyError:
												raise KeyError("invalid input option"    )
						A__							=       create_cipher_map(_A    )
						print(func(_A			,							_A    )    )


if __name__ == "__main__":
						import doctest

						doctest.testmod()
						main()





