

"""simple docstring"""




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE_            = logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_            = {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_            = {
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/tokenizer.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/tokenizer.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/tokenizer.json',
    },
}

SCREAMING_SNAKE_CASE_            = {
    'gpt2': 1024,
    'gpt2-medium': 1024,
    'gpt2-large': 1024,
    'gpt2-xl': 1024,
    'distilgpt2': 1024,
}

class 			a					(		__SCREAMING_SNAKE_CASE			):
						_lowercase        =     VOCAB_FILES_NAMES
						_lowercase        =     PRETRAINED_VOCAB_FILES_MAP
						_lowercase        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_lowercase        =     ['input_ids', 'attention_mask']
						_lowercase        =     GPTaTokenizer


						def __init__(    self  ,   A_=None  ,   A_=None  ,   A_=None  ,   A_="<|endoftext|>"  ,   A_="<|endoftext|>"  ,   A_="<|endoftext|>"  ,   A_=False  ,   **A_  ,   ):







										'''simple docstring'''


										super().__init__(
										    _a  ,   _a  ,   tokenizer_file=_a  ,   unk_token=_a  ,   bos_token=_a  ,   eos_token=_a  ,   add_prefix_space=_a  ,   **_a  ,   )

										_UpperCAmelCase    :						Any							=      kwargs.pop("add_bos_token"  ,   _a		)

										_UpperCAmelCase    :						str							=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
										if pre_tok_state.get("add_prefix_space"  ,   _a		) != add_prefix_space:
														_UpperCAmelCase    :						Union[str, Any]							=      getattr(_a  ,   pre_tok_state.pop("type"		)		)
														_UpperCAmelCase    :						List[str]							=      add_prefix_space
														_UpperCAmelCase    :						Optional[int]							=      pre_tok_class(**_a		)

										_UpperCAmelCase    :						List[str]							=      add_prefix_space


						def     _UpperCAmelCase			(    self  ,   *A_  ,   **A_		):







										'''simple docstring'''


										_UpperCAmelCase    :						List[Any]							=      kwargs.get("is_split_into_words"  ,   _a		)
										assert self.add_prefix_space or not is_split_into_words, (
										    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*_a  ,   **_a		)


						def     _UpperCAmelCase			(    self  ,   *A_  ,   **A_		):







										'''simple docstring'''


										_UpperCAmelCase    :						str							=      kwargs.get("is_split_into_words"  ,   _a		)

										assert self.add_prefix_space or not is_split_into_words, (
										    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*_a  ,   **_a		)


						def     _UpperCAmelCase			(    self  ,   A_  ,   A_ = None		):







										'''simple docstring'''


										_UpperCAmelCase    :						Dict							=      self._tokenizer.model.save(_a  ,   name=_a		)
										return tuple(_a		)





						def     _UpperCAmelCase			(    self  ,   A_		):







										'''simple docstring'''


										_UpperCAmelCase    :						Any							=      []
										for is_user, text in conversation.iter_texts():
														input_ids.extend(self.encode(_a  ,   add_special_tokens=_a		) + [self.eos_token_id]		)

										if len(_a		) > self.model_max_length:
														_UpperCAmelCase    :						Optional[int]							=      input_ids[-self.model_max_length :]
										return input_ids





def 							__SCREAMING_SNAKE_CASE							(			lowerCAmelCase:					int	)				->    int:
				if not isinstance(lowerCAmelCase					, lowerCAmelCase	):
								raise TypeError("Input value must be an 'int' type"	)

				_UpperCAmelCase    :						List[Any]							=      0
				while number:
								position += 1
								number >>= 1

				return position


if __name__ == "__main__":
					import doctest

					doctest.testmod()


