



'''simple docstring'''

import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
				if is_torch_available():
								import torch
				if is_tf_available():
								import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase     		=   logging.get_logger(__name__)

UpperCAmelCase     		=   {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase     		=   {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase     		=   {
    'Salesforce/codegen-350M-mono': 2048,
}

class __snake_case(				_lowerCAmelCase ):


						'''simple docstring'''



						UpperCAmelCase							:   Union[str, Any]									=		VOCAB_FILES_NAMES
						UpperCAmelCase							:   List[Any]									=		PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase							:   Tuple									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase							:   str									=		["input_ids", "attention_mask"]
						UpperCAmelCase							:   int									=		CodeGenTokenizer


						def __init__( self     ,	A_=None     ,	A_=None     ,	A_=None     ,	A_="<|endoftext|>"     ,	A_="<|endoftext|>"     ,	A_="<|endoftext|>"     ,	A_=False     ,	**A_     ,	)     ->     Tuple:
							super().__init__(
							    A_     ,	A_     ,	tokenizer_file=A_     ,	unk_token=A_     ,	bos_token=A_     ,	eos_token=A_     ,	add_prefix_space=A_     ,	**A_     ,	)

							if kwargs.pop("""add_bos_token"""     ,	A_				):
								lowerCAmelCase   =     kwargs.pop("""name_or_path"""     ,	""""""				)
								raise ValueError(
								    """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
								    """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
								    f'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
								    f'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
								    """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
								    """ so that the fast tokenizer works correctly."""				)

							lowerCAmelCase   =     json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
							if pre_tok_state.get("""add_prefix_space"""     ,	A_				) != add_prefix_space:
								lowerCAmelCase   =     getattr(A_     ,	pre_tok_state.pop("""type"""				)				)
								lowerCAmelCase   =     add_prefix_space
								lowerCAmelCase   =     pre_tok_class(**A_				)

							lowerCAmelCase   =     add_prefix_space


						def     __snake_case   ( self     ,	*A_     ,	**A_				)     ->     BatchEncoding:
							lowerCAmelCase   =     kwargs.get("""is_split_into_words"""     ,	A_				)
							assert self.add_prefix_space or not is_split_into_words, (
							    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
							    "to use it with pretokenized inputs."
							)

							return super()._batch_encode_plus(*A_     ,	**A_				)


						def     __snake_case   ( self     ,	*A_     ,	**A_				)     ->     BatchEncoding:
							lowerCAmelCase   =     kwargs.get("""is_split_into_words"""     ,	A_				)

							assert self.add_prefix_space or not is_split_into_words, (
							    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
							    "to use it with pretokenized inputs."
							)

							return super()._encode_plus(*A_     ,	**A_				)


						def     __snake_case   ( self     ,	A_     ,	A_ = None				)     ->     Tuple[str]:
							lowerCAmelCase   =     self._tokenizer.model.save(A_     ,	name=A_				)
							return tuple(A_				)


						def     __snake_case   ( self     ,	A_     ,	A_ = False     ,	A_ = None     ,	A_ = None     ,	**A_     ,	)     ->     str:
							lowerCAmelCase   =     super().decode(
							    token_ids=A_     ,	skip_special_tokens=A_     ,	clean_up_tokenization_spaces=A_     ,	**A_     ,	)

							if truncate_before_pattern is not None and len(A_				) > 0:
								lowerCAmelCase   =     self.truncate(A_     ,	A_				)

							return decoded_text




						def     __snake_case   ( self     ,	A_     ,	A_				)     ->     Tuple:
							def find_re(A_     ,	A_     ,	A_				):
								lowerCAmelCase   =     pattern.search(A_     ,	A_				)
								return m.start() if m else -1

							lowerCAmelCase   =     [re.compile(A_     ,	re.MULTILINE				) for pattern in truncate_before_pattern]

							lowerCAmelCase   =     list(re.finditer("""^print"""     ,	A_     ,	re.MULTILINE				)				)

							if len(A_				) > 1:
								lowerCAmelCase   =     completion[: prints[1].start()]

							lowerCAmelCase   =     list(re.finditer("""^def"""     ,	A_     ,	re.MULTILINE				)				)

							if len(A_				) > 1:
								lowerCAmelCase   =     completion[: defs[1].start()]

							lowerCAmelCase   =     0

							lowerCAmelCase   =     [
							    pos for pos in [find_re(A_     ,	A_     ,	A_				) for terminal in terminals] if pos != -1
							]

							if len(A_				) > 0:
								return completion[: min(A_				)]
							else:
								return completion




'''simple docstring'''

from __future__ import annotations

class __snake_case:


						'''simple docstring'''



						def __init__( self     ,	A_ = 0				)     ->     Dict:
							lowerCAmelCase   =     key


						def     __snake_case   ( self     ,	A_     ,	A_				)     ->     list[str]:
							assert isinstance(A_     ,	A_				) and isinstance(A_     ,	A_				)

							lowerCAmelCase   =     key or self.__key or 1

							# make sure key is an appropriate size
							key %= 255

							return [chr(ord(A_				) ^ key				) for ch in content]


						def     __snake_case   ( self     ,	A_     ,	A_				)     ->     list[str]:
							assert isinstance(A_     ,	A_				) and isinstance(A_     ,	A_				)

							lowerCAmelCase   =     key or self.__key or 1

							# make sure key is an appropriate size
							key %= 255

							return [chr(ord(A_				) ^ key				) for ch in content]


						def     __snake_case   ( self     ,	A_     ,	A_ = 0				)     ->     str:
							assert isinstance(A_     ,	A_				) and isinstance(A_     ,	A_				)

							lowerCAmelCase   =     key or self.__key or 1

							# make sure key can be any size
							while key > 255:
								key -= 255

							# This will be returned
							lowerCAmelCase   =     """"""

							for ch in content:
								ans += chr(ord(A_				) ^ key				)

							return ans


						def     __snake_case   ( self     ,	A_     ,	A_ = 0				)     ->     str:
							assert isinstance(A_     ,	A_				) and isinstance(A_     ,	A_				)

							lowerCAmelCase   =     key or self.__key or 1

							# make sure key can be any size
							while key > 255:
								key -= 255

							# This will be returned
							lowerCAmelCase   =     """"""

							for ch in content:
								ans += chr(ord(A_				) ^ key				)

							return ans


						def     __snake_case   ( self     ,	A_     ,	A_ = 0				)     ->     bool:
							assert isinstance(A_     ,	A_				) and isinstance(A_     ,	A_				)

							try:
								with open(A_				) as fin, open("""encrypt.out"""     ,	"""w+"""				) as fout:
									# actual encrypt-process
									for line in fin:
										fout.write(self.encrypt_string(A_     ,	A_				)				)

							except OSError:
								return False

							return True




						def     __snake_case   ( self     ,	A_     ,	A_				)     ->     bool:
							assert isinstance(A_     ,	A_				) and isinstance(A_     ,	A_				)

							try:
								with open(A_				) as fin, open("""decrypt.out"""     ,	"""w+"""				) as fout:
									# actual encrypt-process
									for line in fin:
										fout.write(self.decrypt_string(A_     ,	A_				)				)

							except OSError:
								return False

							return True


# Tests
# crypt = XORCipher()
# key = 67

# # test encrypt
# print(crypt.encrypt("hallo welt",key))
# # test decrypt
# print(crypt.decrypt(crypt.encrypt("hallo welt",key), key))

# # test encrypt_string
# print(crypt.encrypt_string("hallo welt",key))

# # test decrypt_string
# print(crypt.decrypt_string(crypt.encrypt_string("hallo welt",key),key))

# if (crypt.encrypt_file("test.txt",key)):
#       print("encrypt successful")
# else:
#       print("encrypt unsuccessful")

# if (crypt.decrypt_file("encrypt.out",key)):
#       print("decrypt successful")
# else:
#       print("decrypt unsuccessful")