

"""simple docstring"""




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast

class 			a					(		unittest.TestCase			):


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						Optional[int]							=      mock.Mock()
										_UpperCAmelCase    :						int							=      500
										_UpperCAmelCase    :						List[str]							=      {}
										_UpperCAmelCase    :						Any							=      HTTPError
										_UpperCAmelCase    :						List[str]							=      {}

										# Download this model to make sure it's in the cache.
										_UpperCAmelCase    :						Dict							=      BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"		)

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("requests.Session.request"  ,   return_value=__snake_case		) as mock_head:
														_UpperCAmelCase    :						List[Any]							=      BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"		)
														# This check we did call the fake head request
														mock_head.assert_called()


						@require_tokenizers
						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						str							=      mock.Mock()
										_UpperCAmelCase    :						List[str]							=      500
										_UpperCAmelCase    :						List[str]							=      {}
										_UpperCAmelCase    :						Union[str, Any]							=      HTTPError
										_UpperCAmelCase    :						int							=      {}

										# Download this model to make sure it's in the cache.
										_UpperCAmelCase    :						List[Any]							=      GPTaTokenizerFast.from_pretrained("gpt2"		)

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("requests.Session.request"  ,   return_value=__snake_case		) as mock_head:
														_UpperCAmelCase    :						Optional[Any]							=      GPTaTokenizerFast.from_pretrained("gpt2"		)
														# This check we did call the fake head request
														mock_head.assert_called()


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										try:
														_UpperCAmelCase    :						Dict							=      tempfile.mktemp()
														with open(__snake_case  ,   "wb"		) as f:
																		http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"  ,   __snake_case		)

														_UpperCAmelCase    :						Optional[int]							=      AlbertTokenizer.from_pretrained(__snake_case		)
										finally:
														os.remove(__snake_case		)

										# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
										# the current folder and have the right name.
										if os.path.isfile("tokenizer.json"		):
														# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
														return
										try:
														with open("tokenizer.json"  ,   "wb"		) as f:
																		http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"  ,   __snake_case		)
														_UpperCAmelCase    :						int							=      AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"		)
														# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
														self.assertEqual(tokenizer.vocab_size  ,   1000		)
														# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

										finally:
														os.remove("tokenizer.json"		)





						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						int							=      AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"		)







@is_staging_test
class 			a					(		unittest.TestCase			):
						_lowercase        =     ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


						@classmethod
						def     _UpperCAmelCase			(    cls		):







										'''simple docstring'''


										_UpperCAmelCase    :						Any							=      TOKEN
										HfFolder.save_token(__snake_case		)


						@classmethod
						def     _UpperCAmelCase			(    cls		):







										'''simple docstring'''


										try:
														delete_repo(token=cls._token  ,   repo_id="test-tokenizer"		)
										except HTTPError:
														pass

										try:
														delete_repo(token=cls._token  ,   repo_id="valid_org/test-tokenizer-org"		)
										except HTTPError:
														pass

										try:
														delete_repo(token=cls._token  ,   repo_id="test-dynamic-tokenizer"		)
										except HTTPError:
														pass


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										with tempfile.TemporaryDirectory() as tmp_dir:
														_UpperCAmelCase    :						int							=      os.path.join(__snake_case  ,   "vocab.txt"		)
														with open(__snake_case  ,   "w"  ,   encoding="utf-8"		) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
														_UpperCAmelCase    :						str							=      BertTokenizer(__snake_case		)

										tokenizer.push_to_hub("test-tokenizer"  ,   use_auth_token=self._token		)
										_UpperCAmelCase    :						List[Any]							=      BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'		)
										self.assertDictEqual(new_tokenizer.vocab  ,   tokenizer.vocab		)

										# Reset repo
										delete_repo(token=self._token  ,   repo_id="test-tokenizer"		)

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(__snake_case  ,   repo_id="test-tokenizer"  ,   push_to_hub=__snake_case  ,   use_auth_token=self._token		)

										_UpperCAmelCase    :						Optional[int]							=      BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'		)
										self.assertDictEqual(new_tokenizer.vocab  ,   tokenizer.vocab		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										with tempfile.TemporaryDirectory() as tmp_dir:
														_UpperCAmelCase    :						Optional[Any]							=      os.path.join(__snake_case  ,   "vocab.txt"		)
														with open(__snake_case  ,   "w"  ,   encoding="utf-8"		) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
														_UpperCAmelCase    :						Dict							=      BertTokenizer(__snake_case		)

										tokenizer.push_to_hub("valid_org/test-tokenizer-org"  ,   use_auth_token=self._token		)
										_UpperCAmelCase    :						List[str]							=      BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"		)
										self.assertDictEqual(new_tokenizer.vocab  ,   tokenizer.vocab		)

										# Reset repo
										delete_repo(token=self._token  ,   repo_id="valid_org/test-tokenizer-org"		)

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(
														    __snake_case  ,   repo_id="valid_org/test-tokenizer-org"  ,   push_to_hub=__snake_case  ,   use_auth_token=self._token		)

										_UpperCAmelCase    :						List[str]							=      BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"		)
										self.assertDictEqual(new_tokenizer.vocab  ,   tokenizer.vocab		)





						@require_tokenizers
						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										CustomTokenizer.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
														_UpperCAmelCase    :						Optional[Any]							=      os.path.join(__snake_case  ,   "vocab.txt"		)
														with open(__snake_case  ,   "w"  ,   encoding="utf-8"		) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
														_UpperCAmelCase    :						Tuple							=      CustomTokenizer(__snake_case		)

										# No fast custom tokenizer
										tokenizer.push_to_hub("test-dynamic-tokenizer"  ,   use_auth_token=self._token		)

										_UpperCAmelCase    :						Any							=      AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'  ,   trust_remote_code=__snake_case		)
										# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,   "CustomTokenizer"		)

										# Fast and slow custom tokenizer
										CustomTokenizerFast.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
														_UpperCAmelCase    :						Union[str, Any]							=      os.path.join(__snake_case  ,   "vocab.txt"		)
														with open(__snake_case  ,   "w"  ,   encoding="utf-8"		) as vocab_writer:
																		vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)

														_UpperCAmelCase    :						Dict							=      BertTokenizerFast.from_pretrained(__snake_case		)
														bert_tokenizer.save_pretrained(__snake_case		)
														_UpperCAmelCase    :						int							=      CustomTokenizerFast.from_pretrained(__snake_case		)

										tokenizer.push_to_hub("test-dynamic-tokenizer"  ,   use_auth_token=self._token		)

										_UpperCAmelCase    :						Union[str, Any]							=      AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'  ,   trust_remote_code=__snake_case		)
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,   "CustomTokenizerFast"		)
										_UpperCAmelCase    :						str							=      AutoTokenizer.from_pretrained(
										    f'{USER}/test-dynamic-tokenizer'  ,   use_fast=__snake_case  ,   trust_remote_code=__snake_case		)
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__  ,   "CustomTokenizer"		)







class 			a					(		unittest.TestCase			):


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						Any							=      Trie()
										trie.add("Hello 友達"		)
										self.assertEqual(trie.data  ,   {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}		)
										trie.add("Hello"		)
										trie.data
										self.assertEqual(trie.data  ,   {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						List[str]							=      Trie()
										self.assertEqual(trie.split("[CLS] This is a extra_id_100"		)  ,   ["[CLS] This is a extra_id_100"]		)
										trie.add("[CLS]"		)
										trie.add("extra_id_1"		)
										trie.add("extra_id_100"		)
										self.assertEqual(trie.split("[CLS] This is a extra_id_100"		)  ,   ["[CLS]", " This is a ", "extra_id_100"]		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						Tuple							=      Trie()
										trie.add("A"		)
										self.assertEqual(trie.split("ABC"		)  ,   ["A", "BC"]		)
										self.assertEqual(trie.split("BCA"		)  ,   ["BC", "A"]		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						Union[str, Any]							=      Trie()
										trie.add("TOKEN]"		)
										trie.add("[SPECIAL_TOKEN]"		)
										self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"		)  ,   ["This is something ", "[SPECIAL_TOKEN]"]		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						List[str]							=      Trie()
										trie.add("A"		)
										trie.add("P"		)
										trie.add("[SPECIAL_TOKEN]"		)
										self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"		)  ,   ["This is something ", "[SPECIAL_TOKEN]"]		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						int							=      Trie()
										trie.add("AB"		)
										trie.add("B"		)
										trie.add("C"		)
										self.assertEqual(trie.split("ABC"		)  ,   ["AB", "C"]		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						Any							=      Trie()
										trie.add("ABC"		)
										trie.add("B"		)
										trie.add("CD"		)
										self.assertEqual(trie.split("ABCD"		)  ,   ["ABC", "D"]		)





						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						Optional[Any]							=      Trie()
										_UpperCAmelCase    :						Union[str, Any]							=      trie.cut_text("ABC"  ,   [0, 0, 2, 1, 2, 3]		)
										self.assertEqual(__snake_case  ,   ["AB", "C"]		)





from typing import List, Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_            = logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_            = {
    'huggingface/autoformer-tourism-monthly': 'https://huggingface.co/huggingface/autoformer-tourism-monthly/resolve/main/config.json',
}

class 			a					(		UpperCAmelCase			):
						_lowercase        =     "autoformer"
						_lowercase        =     {
						    "hidden_size": "d_model",
						    "num_attention_heads": "encoder_attention_heads",
						    "num_hidden_layers": "encoder_layers",
						}


						def __init__(    self  ,   A_ = None  ,   A_ = None  ,   A_ = "student_t"  ,   A_ = "nll"  ,   A_ = 1  ,   A_ = [1, 2, 3, 4, 5, 6, 7]  ,   A_ = True  ,   A_ = 0  ,   A_ = 0  ,   A_ = 0  ,   A_ = 0  ,   A_ = None  ,   A_ = None  ,   A_ = 64  ,   A_ = 2  ,   A_ = 2  ,   A_ = 2  ,   A_ = 2  ,   A_ = 32  ,   A_ = 32  ,   A_ = "gelu"  ,   A_ = 0.1  ,   A_ = 0.1  ,   A_ = 0.1  ,   A_ = 0.1  ,   A_ = 0.1  ,   A_ = 100  ,   A_ = 0.02  ,   A_ = True  ,   A_=True  ,   A_ = 10  ,   A_ = 25  ,   A_ = 3  ,   **A_  ,   ):







										'''simple docstring'''


										_UpperCAmelCase    :						List[Any]							=      prediction_length
										_UpperCAmelCase    :						Dict							=      context_length if context_length is not None else prediction_length
										_UpperCAmelCase    :						Tuple							=      distribution_output
										_UpperCAmelCase    :						List[Any]							=      loss
										_UpperCAmelCase    :						Optional[Any]							=      input_size
										_UpperCAmelCase    :						int							=      num_time_features
										_UpperCAmelCase    :						str							=      lags_sequence
										_UpperCAmelCase    :						Union[str, Any]							=      scaling
										_UpperCAmelCase    :						Union[str, Any]							=      num_dynamic_real_features
										_UpperCAmelCase    :						int							=      num_static_real_features
										_UpperCAmelCase    :						int							=      num_static_categorical_features
										if cardinality is not None and num_static_categorical_features > 0:
														if len(A_		) != num_static_categorical_features:
																		raise ValueError(
																		    "The cardinality should be a list of the same length as `num_static_categorical_features`"		)
														_UpperCAmelCase    :						Union[str, Any]							=      cardinality
										else:
														_UpperCAmelCase    :						Any							=      [0]
										if embedding_dimension is not None and num_static_categorical_features > 0:
														if len(A_		) != num_static_categorical_features:
																		raise ValueError(
																		    "The embedding dimension should be a list of the same length as `num_static_categorical_features`"		)
														_UpperCAmelCase    :						int							=      embedding_dimension
										else:
														_UpperCAmelCase    :						int							=      [min(50  ,   (cat + 1) // 2		) for cat in self.cardinality]
										_UpperCAmelCase    :						Union[str, Any]							=      num_parallel_samples

										# Transformer architecture configuration
										_UpperCAmelCase    :						Union[str, Any]							=      input_size * len(self.lags_sequence		) + self._number_of_features
										_UpperCAmelCase    :						int							=      d_model
										_UpperCAmelCase    :						Any							=      encoder_attention_heads
										_UpperCAmelCase    :						str							=      decoder_attention_heads
										_UpperCAmelCase    :						Union[str, Any]							=      encoder_ffn_dim
										_UpperCAmelCase    :						Any							=      decoder_ffn_dim
										_UpperCAmelCase    :						Any							=      encoder_layers
										_UpperCAmelCase    :						Optional[int]							=      decoder_layers

										_UpperCAmelCase    :						Optional[Any]							=      dropout
										_UpperCAmelCase    :						Union[str, Any]							=      attention_dropout
										_UpperCAmelCase    :						Dict							=      activation_dropout
										_UpperCAmelCase    :						Dict							=      encoder_layerdrop
										_UpperCAmelCase    :						int							=      decoder_layerdrop

										_UpperCAmelCase    :						Union[str, Any]							=      activation_function
										_UpperCAmelCase    :						str							=      init_std

										_UpperCAmelCase    :						Tuple							=      use_cache

										# Autoformer
										_UpperCAmelCase    :						str							=      label_length
										_UpperCAmelCase    :						Any							=      moving_average
										_UpperCAmelCase    :						Optional[Any]							=      autocorrelation_factor

										super().__init__(is_encoder_decoder=A_  ,   **A_		)





						@property
						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										return (
										    sum(self.embedding_dimension		)
										    + self.num_dynamic_real_features
										    + self.num_time_features
										    + self.num_static_real_features
										    + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
										)


