




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_snake_case				:				Tuple       							=		logging.get_logger(__name__)

_snake_case				:				List[Any]       							=		{"vocab_file": "spiece.model"}

_snake_case				:				Optional[int]       							=		{
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

_snake_case				:				Optional[int]       							=		{
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
_snake_case				:				int       							=		0
_snake_case				:				Optional[Any]       							=		1
_snake_case				:				List[Any]       							=		2
_snake_case				:				str       							=		3
_snake_case				:				Optional[int]       							=		4

class       A   (		SCREAMING_SNAKE_CASE__   ):
							lowercase_          =    VOCAB_FILES_NAMES
							lowercase_          =    PRETRAINED_VOCAB_FILES_MAP
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_          =    '''left'''







							def __init__(  self	:  Union[str, Any]							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  List[str]=False							,      lowerCAmelCase_	:  Union[str, Any]=True							,      lowerCAmelCase_	:  Optional[int]=False							,      lowerCAmelCase_	:  int="<s>"							,      lowerCAmelCase_	:  Any="</s>"							,      lowerCAmelCase_	:  Dict="<unk>"							,      lowerCAmelCase_	:  str="<sep>"							,      lowerCAmelCase_	:  Optional[Any]="<pad>"							,      lowerCAmelCase_	:  Dict="<cls>"							,      lowerCAmelCase_	:  Optional[Any]="<mask>"							,      lowerCAmelCase_	:  Optional[int]=["<eop>", "<eod>"]							,      lowerCAmelCase_	:  Union[str, Any] = None							,      **lowerCAmelCase_	:  Any							,      )      ->							Tuple:


														"""simple docstring"""


														_a     					=    AddedToken(A__							,      lstrip=A__							,      rstrip=A__ ) if isinstance(A__							,      A__ ) else mask_token

														_a     					=    {} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    do_lower_case=A__							,      remove_space=A__							,      keep_accents=A__							,      bos_token=A__							,      eos_token=A__							,      unk_token=A__							,      sep_token=A__							,      pad_token=A__							,      cls_token=A__							,      mask_token=A__							,      additional_special_tokens=A__							,      sp_model_kwargs=self.sp_model_kwargs							,      **A__							,      )

														_a     					=    3

														_a     					=    do_lower_case
														_a     					=    remove_space
														_a     					=    keep_accents
														_a     					=    vocab_file

														_a     					=    spm.SentencePieceProcessor(**self.sp_model_kwargs )
														self.sp_model.Load(A__ )







							@property
							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							Union[str, Any]:


														"""simple docstring"""


														return len(self.sp_model )







							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							int:


														"""simple docstring"""


														_a     					=    {self.convert_ids_to_tokens(A__ ): i for i in range(self.vocab_size )}
														vocab.update(self.added_tokens_encoder )
														return vocab







							def __getstate__(  self	:  List[Any] )      ->							Dict:


														"""simple docstring"""


														_a     					=    self.__dict__.copy()
														_a     					=    None
														return state







							def __setstate__(  self	:  Tuple							,      lowerCAmelCase_	:  Dict )      ->							Dict:


														"""simple docstring"""


														_a     					=    d

														# for backward compatibility
														if not hasattr(self							,      '''sp_model_kwargs''' ):
																					_a     					=    {}

														_a     					=    spm.SentencePieceProcessor(**self.sp_model_kwargs )
														self.sp_model.Load(self.vocab_file )







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  List[Any] )      ->							Optional[Any]:


														"""simple docstring"""


														if self.remove_space:
																					_a     					=    ''' '''.join(inputs.strip().split() )
														else:
																					_a     					=    inputs
														_a     					=    outputs.replace('''``'''							,      '''"''' ).replace('''\'\''''							,      '''"''' )

														if not self.keep_accents:
																					_a     					=    unicodedata.normalize('''NFKD'''							,      A__ )
																					_a     					=    ''''''.join([c for c in outputs if not unicodedata.combining(A__ )] )
														if self.do_lower_case:
																					_a     					=    outputs.lower()

														return outputs







							def        __lowerCAmelCase      (  self	:  Union[str, Any]							,      lowerCAmelCase_	:  Dict )      ->							int:


														"""simple docstring"""


														_a     					=    self.preprocess_text(A__ )
														_a     					=    self.sp_model.encode(A__							,      out_type=A__ )
														_a     					=    []
														for piece in pieces:
																					if len(A__ ) > 1 and piece[-1] == str(''',''' ) and piece[-2].isdigit():
																												_a     					=    self.sp_model.EncodeAsPieces(piece[:-1].replace(A__							,      '''''' ) )
																												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																			if len(cur_pieces[0] ) == 1:
																																										_a     					=    cur_pieces[1:]
																																			else:
																																										_a     					=    cur_pieces[0][1:]
																												cur_pieces.append(piece[-1] )
																												new_pieces.extend(A__ )
																					else:
																												new_pieces.append(A__ )

														return new_pieces







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  List[str] )      ->							Tuple:


														"""simple docstring"""


														return self.sp_model.PieceToId(A__ )







							def        __lowerCAmelCase      (  self	:  Union[str, Any]							,      lowerCAmelCase_	:  Any )      ->							Any:


														"""simple docstring"""


														return self.sp_model.IdToPiece(A__ )







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  Optional[Any] )      ->							str:


														"""simple docstring"""


														_a     					=    ''''''.join(A__ ).replace(A__							,      ''' ''' ).strip()
														return out_string







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  List[Any] = False							,      lowerCAmelCase_	:  Optional[int] = None							,      lowerCAmelCase_	:  Union[str, Any] = True							,      **lowerCAmelCase_	:  List[Any]							,      )      ->							Tuple:


														"""simple docstring"""


														_a     					=    kwargs.pop('''use_source_tokenizer'''							,      A__ )

														_a     					=    self.convert_ids_to_tokens(A__							,      skip_special_tokens=A__ )

														# To avoid mixing byte-level and unicode for byte-level BPT
														# we need to build string separately for added tokens and byte-level tokens
														# cf. https://github.com/huggingface/transformers/issues/1133
														_a     					=    []
														_a     					=    []
														for token in filtered_tokens:
																					if skip_special_tokens and token in self.all_special_ids:
																												continue
																					if token in self.added_tokens_encoder:
																												if current_sub_text:
																																			sub_texts.append(self.convert_tokens_to_string(A__ ) )
																																			_a     					=    []
																												sub_texts.append(A__ )
																					else:
																												current_sub_text.append(A__ )
														if current_sub_text:
																					sub_texts.append(self.convert_tokens_to_string(A__ ) )

														# Mimic the behavior of the Rust tokenizer:
														# By default, there are no spaces between special tokens
														_a     					=    ''''''.join(A__ )

														_a     					=    (
														    clean_up_tokenization_spaces
														    if clean_up_tokenization_spaces is not None
														    else self.clean_up_tokenization_spaces
														)
														if clean_up_tokenization_spaces:
																					_a     					=    self.clean_up_tokenization(A__ )
																					return clean_text
														else:
																					return text







							def        __lowerCAmelCase      (  self	:  Tuple							,      lowerCAmelCase_	:  Union[str, Any]							,      lowerCAmelCase_	:  Dict = None )      ->							Union[str, Any]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														if token_ids_a is None:
																					return token_ids_a + sep + cls
														return token_ids_a + sep + token_ids_a + sep + cls







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  List[str]							,      lowerCAmelCase_	:  Optional[Any] = None							,      lowerCAmelCase_	:  int = False )      ->							Optional[int]:


														"""simple docstring"""


														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=A__							,      token_ids_a=A__							,      already_has_special_tokens=A__ )

														if token_ids_a is not None:
																					return ([0] * len(A__ )) + [1] + ([0] * len(A__ )) + [1, 1]
														return ([0] * len(A__ )) + [1, 1]







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  Union[str, Any]							,      lowerCAmelCase_	:  Union[str, Any] = None )      ->							Optional[Any]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [2]

														if token_ids_a is None:
																					return len(token_ids_a + sep ) * [0] + cls_segment_id
														return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1] + cls_segment_id







							def        __lowerCAmelCase      (  self	:  Union[str, Any]							,      lowerCAmelCase_	:  Union[str, Any]							,      lowerCAmelCase_	:  Any = None )      ->							Tuple:


														"""simple docstring"""


														if not os.path.isdir(A__ ):
																					logger.error(F'Vocabulary path ({save_directory}) should be a directory' )
																					return
														_a     					=    os.path.join(
														    A__							,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

														if os.path.abspath(self.vocab_file ) != os.path.abspath(A__ ) and os.path.isfile(self.vocab_file ):
																					copyfile(self.vocab_file							,      A__ )
														elif not os.path.isfile(self.vocab_file ):
																					with open(A__							,      '''wb''' ) as fi:
																												_a     					=    self.sp_model.serialized_model_proto()
																												fi.write(A__ )

														return (out_vocab_file,)







'''simple docstring'''


import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


_snake_case				:				int       							=		get_tests_dir('fixtures')
_snake_case				:				Tuple       							=		get_tests_dir('fixtures/dummy_feature_extractor_config.json')
_snake_case				:				Optional[int]       							=		get_tests_dir('fixtures/dummy-config.json')

class       A   (		unittest.TestCase   ):







							def        __lowerCAmelCase      (  self	:  int )      ->							List[Any]:


														"""simple docstring"""


														_a     					=    0







							def        __lowerCAmelCase      (  self	:  List[str] )      ->							int:


														"""simple docstring"""


														_a     					=    AutoFeatureExtractor.from_pretrained('''facebook/wav2vec2-base-960h''' )
														self.assertIsInstance(lowerCAmelCase_							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  str )      ->							Tuple:


														"""simple docstring"""


														_a     					=    AutoFeatureExtractor.from_pretrained(lowerCAmelCase_ )
														self.assertIsInstance(lowerCAmelCase_							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  List[str] )      ->							Any:


														"""simple docstring"""


														with tempfile.TemporaryDirectory() as tmpdirname:
																					_a     					=    WavaVecaConfig()

																					# remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
																					_a     					=    AutoFeatureExtractor.from_pretrained(lowerCAmelCase_ ).to_dict()

																					config_dict.pop('''feature_extractor_type''' )
																					_a     					=    WavaVecaFeatureExtractor(**lowerCAmelCase_ )

																					# save in new folder
																					model_config.save_pretrained(lowerCAmelCase_ )
																					config.save_pretrained(lowerCAmelCase_ )

																					_a     					=    AutoFeatureExtractor.from_pretrained(lowerCAmelCase_ )

																					# make sure private variable is not incorrectly saved
																					_a     					=    json.loads(config.to_json_string() )
																					self.assertTrue('''_processor_class''' not in dict_as_saved )

														self.assertIsInstance(lowerCAmelCase_							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Optional[Any] )      ->							Optional[Any]:


														"""simple docstring"""


														_a     					=    AutoFeatureExtractor.from_pretrained(lowerCAmelCase_ )
														self.assertIsInstance(lowerCAmelCase_							,      lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							str:


														"""simple docstring"""


														with self.assertRaisesRegex(
														    lowerCAmelCase_							,      '''bert-base is not a local folder and is not a valid model identifier''' ):
																					_a     					=    AutoFeatureExtractor.from_pretrained('''bert-base''' )







							def        __lowerCAmelCase      (  self	:  Tuple )      ->							Union[str, Any]:


														"""simple docstring"""


														with self.assertRaisesRegex(
														    lowerCAmelCase_							,      R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)''' ):
																					_a     					=    AutoFeatureExtractor.from_pretrained(lowerCAmelCase_							,      revision='''aaaaaa''' )







							def        __lowerCAmelCase      (  self	:  Any )      ->							Dict:


														"""simple docstring"""


														with self.assertRaisesRegex(
														    lowerCAmelCase_							,      '''hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'''							,      ):
																					_a     					=    AutoFeatureExtractor.from_pretrained('''hf-internal-testing/config-no-model''' )







							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							Any:


														"""simple docstring"""


														with self.assertRaises(lowerCAmelCase_ ):
																					_a     					=    AutoFeatureExtractor.from_pretrained(
																					    '''hf-internal-testing/test_dynamic_feature_extractor''' )
														# If remote code is disabled, we can't load this config.
														with self.assertRaises(lowerCAmelCase_ ):
																					_a     					=    AutoFeatureExtractor.from_pretrained(
																					    '''hf-internal-testing/test_dynamic_feature_extractor'''							,      trust_remote_code=lowerCAmelCase_ )

														_a     					=    AutoFeatureExtractor.from_pretrained(
														    '''hf-internal-testing/test_dynamic_feature_extractor'''							,      trust_remote_code=lowerCAmelCase_ )
														self.assertEqual(feature_extractor.__class__.__name__							,      '''NewFeatureExtractor''' )

														# Test feature extractor can be reloaded.
														with tempfile.TemporaryDirectory() as tmp_dir:
																					feature_extractor.save_pretrained(lowerCAmelCase_ )
																					_a     					=    AutoFeatureExtractor.from_pretrained(lowerCAmelCase_							,      trust_remote_code=lowerCAmelCase_ )
														self.assertEqual(reloaded_feature_extractor.__class__.__name__							,      '''NewFeatureExtractor''' )







							def        __lowerCAmelCase      (  self	:  int )      ->							Optional[Any]:


														"""simple docstring"""


														try:
																					AutoConfig.register('''custom'''							,      lowerCAmelCase_ )
																					AutoFeatureExtractor.register(lowerCAmelCase_							,      lowerCAmelCase_ )
																					# Trying to register something existing in the Transformers library will raise an error
																					with self.assertRaises(lowerCAmelCase_ ):
																												AutoFeatureExtractor.register(lowerCAmelCase_							,      lowerCAmelCase_ )

																					# Now that the config is registered, it can be used as any other config with the auto-API
																					_a     					=    CustomFeatureExtractor.from_pretrained(lowerCAmelCase_ )
																					with tempfile.TemporaryDirectory() as tmp_dir:
																												feature_extractor.save_pretrained(lowerCAmelCase_ )
																												_a     					=    AutoFeatureExtractor.from_pretrained(lowerCAmelCase_ )
																												self.assertIsInstance(lowerCAmelCase_							,      lowerCAmelCase_ )

														finally:
																					if "custom" in CONFIG_MAPPING._extra_content:
																												del CONFIG_MAPPING._extra_content["custom"]
																					if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																												del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]







							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							Any:


														"""simple docstring"""


														class       A   (		_a   ):
																					lowercase_          =    True

														try:
																					AutoConfig.register('''custom'''							,      lowerCAmelCase_ )
																					AutoFeatureExtractor.register(lowerCAmelCase_							,      lowerCAmelCase_ )
																					# If remote code is not set, the default is to use local
																					_a     					=    AutoFeatureExtractor.from_pretrained(
																					    '''hf-internal-testing/test_dynamic_feature_extractor''' )
																					self.assertEqual(feature_extractor.__class__.__name__							,      '''NewFeatureExtractor''' )
																					self.assertTrue(feature_extractor.is_local )

																					# If remote code is disabled, we load the local one.
																					_a     					=    AutoFeatureExtractor.from_pretrained(
																					    '''hf-internal-testing/test_dynamic_feature_extractor'''							,      trust_remote_code=lowerCAmelCase_ )
																					self.assertEqual(feature_extractor.__class__.__name__							,      '''NewFeatureExtractor''' )
																					self.assertTrue(feature_extractor.is_local )

																					# If remote is enabled, we load from the Hub
																					_a     					=    AutoFeatureExtractor.from_pretrained(
																					    '''hf-internal-testing/test_dynamic_feature_extractor'''							,      trust_remote_code=lowerCAmelCase_ )
																					self.assertEqual(feature_extractor.__class__.__name__							,      '''NewFeatureExtractor''' )
																					self.assertTrue(not hasattr(lowerCAmelCase_							,      '''is_local''' ) )

														finally:
																					if "custom" in CONFIG_MAPPING._extra_content:
																												del CONFIG_MAPPING._extra_content["custom"]
																					if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
																												del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]

