





'''simple docstring'''



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =      '''▁'''

_SCREAMING_SNAKE_CASE   =      {'''vocab_file''': '''sentencepiece.bpe.model'''}

_SCREAMING_SNAKE_CASE   =      {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

_SCREAMING_SNAKE_CASE   =      {
    '''xlm-roberta-base''': 5_1_2,
    '''xlm-roberta-large''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_1_2,
}






class 	__lowercase    ( SCREAMING_SNAKE_CASE_  ):







	'''simple docstring'''

	a      :			Optional[int]     =    VOCAB_FILES_NAMES
	a      :			str     =    PRETRAINED_VOCAB_FILES_MAP
	a      :			Any     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a      :			List[str]     =    ["input_ids", "attention_mask"]


	def __init__(self       ,_lowerCamelCase       ,_lowerCamelCase="<s>"       ,_lowerCamelCase="</s>"       ,_lowerCamelCase="</s>"       ,_lowerCamelCase="<s>"       ,_lowerCamelCase="<unk>"       ,_lowerCamelCase="<pad>"       ,_lowerCamelCase="<mask>"       ,_lowerCamelCase = None       ,**_lowerCamelCase       ,)							->							None:







				'''simple docstring'''





				__lowercase			  =							AddedToken(__a       ,lstrip=__a       ,rstrip=__a	) if isinstance(__a       ,__a	) else mask_token

				__lowercase			  =							{} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    bos_token=__a       ,eos_token=__a       ,unk_token=__a       ,sep_token=__a       ,cls_token=__a       ,pad_token=__a       ,mask_token=__a       ,sp_model_kwargs=self.sp_model_kwargs       ,**__a       ,)

				__lowercase			  =							spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.Load(str(__a	)	)
				__lowercase			  =							vocab_file

				# Original fairseq vocab and spm vocab must be "aligned":
				# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
				# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
				# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
				# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

				# Mimic fairseq token-to-id alignment for the first 4 token
				__lowercase			  =							{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

				# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
				__lowercase			  =							1

				__lowercase			  =							len(self.sp_model	) + self.fairseq_offset
				__lowercase			  =							{v: k for k, v in self.fairseq_tokens_to_ids.items()}


	def __getstate__(self	)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							self.__dict__.copy()
				__lowercase			  =							None
				__lowercase			  =							self.sp_model.serialized_model_proto()
				return state


	def __setstate__(self       ,_lowerCamelCase	)							->							List[Any]:







				'''simple docstring'''





				__lowercase			  =							d

				# for backward compatibility
				if not hasattr(self       ,'''sp_model_kwargs'''	):
							__lowercase			  =							{}

				__lowercase			  =							spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.LoadFromSerializedProto(self.sp_model_proto	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							List[int]:







				'''simple docstring'''





				if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				__lowercase			  =							[self.cls_token_id]
				__lowercase			  =							[self.sep_token_id]
				return cls + token_ids_a + sep + sep + token_ids_a + sep


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None       ,_lowerCamelCase = False	)							->							List[int]:







				'''simple docstring'''





				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=__a       ,token_ids_a=__a       ,already_has_special_tokens=__a	)

				if token_ids_a is None:
							return [1] + ([0] * len(__a	)) + [1]
				return [1] + ([0] * len(__a	)) + [1, 1] + ([0] * len(__a	)) + [1]


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							List[int]:







				'''simple docstring'''





				__lowercase			  =							[self.sep_token_id]
				__lowercase			  =							[self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep	) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]


	@property
	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				return len(self.sp_model	) + self.fairseq_offset + 1  # Add the <mask> token


	def   _UpperCAmelCase       (self	)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							{self.convert_ids_to_tokens(__a	): i for i in range(self.vocab_size	)}
				vocab.update(self.added_tokens_encoder	)
				return vocab


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							List[str]:







				'''simple docstring'''





				return self.sp_model.encode(__a       ,out_type=__a	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Any:







				'''simple docstring'''





				if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
				__lowercase			  =							self.sp_model.PieceToId(__a	)

				# Need to return unknown token if the SP model returned 0
				return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Dict:







				'''simple docstring'''





				if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
				return self.sp_model.IdToPiece(index - self.fairseq_offset	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Any:







				'''simple docstring'''





				__lowercase			  =							''.join(__a	).replace(__a       ,''' '''	).strip()
				return out_string


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							Tuple[str]:







				'''simple docstring'''





				if not os.path.isdir(__a	):
							logger.error(f"Vocabulary path ({save_directory}) should be a directory"	)
							return
				__lowercase			  =							os.path.join(
				    __a       ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

				if os.path.abspath(self.vocab_file	) != os.path.abspath(__a	) and os.path.isfile(self.vocab_file	):
							copyfile(self.vocab_file       ,__a	)
				elif not os.path.isfile(self.vocab_file	):
							with open(__a       ,'''wb'''	) as fi:
										__lowercase			  =							self.sp_model.serialized_model_proto()
										fi.write(__a	)

				return (out_vocab_file,)












'''simple docstring'''



import os
import random
import sys

from . import cryptomath_module as cryptoMath  # noqa: N812
from . import rabin_miller as rabinMiller  # noqa: N812




def 				_lowerCAmelCase  (			):
			print('''Making key files...'''			)
			make_key_files('''rsa'''						,    1_0_2_4			)
			print('''Key files generation successful.'''			)




def 				_lowerCAmelCase  (			lowerCamelCase_     :					int			):
			print('''Generating prime p...'''			)
			__lowercase			  =							rabinMiller.generate_large_prime(lowerCamelCase_			)
			print('''Generating prime q...'''			)
			__lowercase			  =							rabinMiller.generate_large_prime(lowerCamelCase_			)
			__lowercase			  =							p * q

			print('''Generating e that is relatively prime to (p - 1) * (q - 1)...'''			)
			while True:
						__lowercase			  =							random.randrange(2 ** (key_size - 1)						,    2 ** (key_size)			)
						if cryptoMath.gcd(lowerCamelCase_						,    (p - 1) * (q - 1)			) == 1:
									break

			print('''Calculating d that is mod inverse of e...'''			)
			__lowercase			  =							cryptoMath.find_mod_inverse(lowerCamelCase_						,    (p - 1) * (q - 1)			)

			__lowercase			  =							(n, e)
			__lowercase			  =							(n, d)
			return (public_key, private_key)




def 				_lowerCAmelCase  (			lowerCamelCase_     :					str						,    lowerCamelCase_     :					int			):
			if os.path.exists(f"{name}_pubkey.txt"			) or os.path.exists(f"{name}_privkey.txt"			):
						print('''\nWARNING:'''			)
						print(
						    f"\"{name}_pubkey.txt\" or \"{name}_privkey.txt\" already exists. \n"
						    '''Use a different name or delete these files and re-run this program.'''			)
						sys.exit()

			__lowercase  ,						__lowercase			  =							generate_key(lowerCamelCase_			)
			print(f"\nWriting public key to file {name}_pubkey.txt..."			)
			with open(f"{name}_pubkey.txt"						,    '''w'''			) as out_file:
						out_file.write(f"{key_size},{public_key[0]},{public_key[1]}"			)

			print(f"Writing private key to file {name}_privkey.txt..."			)
			with open(f"{name}_privkey.txt"						,    '''w'''			) as out_file:
						out_file.write(f"{key_size},{private_key[0]},{private_key[1]}"			)


if __name__ == "__main__":
	main()





