


import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin


class 						lowerCamelCase       (__lowerCamelCase							,     unittest.TestCase      ):







							"""simple docstring"""




							UpperCAmelCase_      						=   ProphetNetTokenizer
							UpperCAmelCase_      						=   False






							def  A_     ( self     :					List[str]						)				->			Optional[Any]:





														"""simple docstring"""
														super().setUp()

														SCREAMING_SNAKE_CASE__       :      Any     =						[
														    "[UNK]",
														    "[CLS]",
														    "[SEP]",
														    "[PAD]",
														    "[MASK]",
														    "want",
														    "##want",
														    "##ed",
														    "wa",
														    "un",
														    "runn",
														    "##ing",
														    ",",
														    "low",
														    "lowest",
														]
														SCREAMING_SNAKE_CASE__       :      str     =						os.path.join(self.tmpdirname,			VOCAB_FILES_NAMES["vocab_file"]						)
														with open(self.vocab_file,			"w",			encoding="utf-8"						) as vocab_writer:
																					vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)






							def  A_     ( self     :					int,			_UpperCAmelCase     :					List[str]						)				->			List[Any]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      List[Any]     =						"UNwant\u00E9d,running"
														SCREAMING_SNAKE_CASE__       :      Dict     =						"unwanted, running"
														return input_text, output_text






							def  A_     ( self     :					str						)				->			List[str]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						self.tokenizer_class(self.vocab_file						)

														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						tokenizer.tokenize("UNwant\u00E9d,running"						)
														self.assertListEqual(_UpperCAmelCase,			["un", "##want", "##ed", ",", "runn", "##ing"]						)
														self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						),			[9, 6, 7, 1_2, 1_0, 1_1]						)






							def  A_     ( self     :					int						)				->			Optional[Any]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      Dict     =						BasicTokenizer()

														self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						),			["ah", "\u535A", "\u63A8", "zz"]						)






							def  A_     ( self     :					Optional[int]						)				->			Dict:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      int     =						BasicTokenizer(do_lower_case=_UpperCAmelCase						)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						),			["hello", "!", "how", "are", "you", "?"]						)
														self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						),			["hello"]						)






							def  A_     ( self     :					Any						)				->			int:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						BasicTokenizer(do_lower_case=_UpperCAmelCase,			strip_accents=_UpperCAmelCase						)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						),			["hällo", "!", "how", "are", "you", "?"]						)
														self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						),			["h\u00E9llo"]						)






							def  A_     ( self     :					Optional[int]						)				->			Optional[int]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      List[Any]     =						BasicTokenizer(do_lower_case=_UpperCAmelCase,			strip_accents=_UpperCAmelCase						)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						),			["hallo", "!", "how", "are", "you", "?"]						)
														self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						),			["hello"]						)






							def  A_     ( self     :					List[Any]						)				->			List[str]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      str     =						BasicTokenizer(do_lower_case=_UpperCAmelCase						)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						),			["hallo", "!", "how", "are", "you", "?"]						)
														self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						),			["hello"]						)






							def  A_     ( self     :					Optional[int]						)				->			List[str]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      Optional[int]     =						BasicTokenizer(do_lower_case=_UpperCAmelCase						)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						),			["HeLLo", "!", "how", "Are", "yoU", "?"]						)






							def  A_     ( self     :					Optional[int]						)				->			Tuple:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      Any     =						BasicTokenizer(do_lower_case=_UpperCAmelCase,			strip_accents=_UpperCAmelCase						)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						),			["HäLLo", "!", "how", "Are", "yoU", "?"]						)






							def  A_     ( self     :					List[str]						)				->			Union[str, Any]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      int     =						BasicTokenizer(do_lower_case=_UpperCAmelCase,			strip_accents=_UpperCAmelCase						)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						),			["HaLLo", "!", "how", "Are", "yoU", "?"]						)






							def  A_     ( self     :					List[str]						)				->			Dict:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      Any     =						BasicTokenizer(do_lower_case=_UpperCAmelCase,			never_split=["[UNK]"]						)

														self.assertListEqual(
														    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						),			["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)






							def  A_     ( self     :					Dict						)				->			Any:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      Optional[int]     =						["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

														SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						{}
														for i, token in enumerate(_UpperCAmelCase						):
																					SCREAMING_SNAKE_CASE__       :      Dict     =						i
														SCREAMING_SNAKE_CASE__       :      Dict     =						WordpieceTokenizer(vocab=_UpperCAmelCase,			unk_token="[UNK]"						)

														self.assertListEqual(tokenizer.tokenize(""						),			[]						)

														self.assertListEqual(tokenizer.tokenize("unwanted running"						),			["un", "##want", "##ed", "runn", "##ing"]						)

														self.assertListEqual(tokenizer.tokenize("unwantedX running"						),			["[UNK]", "runn", "##ing"]						)






							@require_torch
							def  A_     ( self     :					List[Any]						)				->			Tuple:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						self.tokenizer_class.from_pretrained("microsoft/prophetnet-large-uncased"						)

														SCREAMING_SNAKE_CASE__       :      str     =						["A long paragraph for summarization.", "Another paragraph for summarization."]
														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						[1_0_3_7, 2_1_4_6, 2_0_4_2_3, 2_0_0_5, 7_6_8_0, 7_8_4_9, 3_9_8_9, 1_0_1_2, 1_0_2]
														SCREAMING_SNAKE_CASE__       :      int     =						tokenizer(_UpperCAmelCase,			padding=_UpperCAmelCase,			return_tensors="pt"						)
														self.assertIsInstance(_UpperCAmelCase,			_UpperCAmelCase						)
														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						list(batch.input_ids.numpy()[0]						)
														self.assertListEqual(_UpperCAmelCase,			_UpperCAmelCase						)

														self.assertEqual((2, 9),			batch.input_ids.shape						)
														self.assertEqual((2, 9),			batch.attention_mask.shape						)






							def  A_     ( self     :					List[Any]						)				->			Any:





														"""simple docstring"""
														self.assertTrue(_is_whitespace(" "						)						)
														self.assertTrue(_is_whitespace("\t"						)						)
														self.assertTrue(_is_whitespace("\r"						)						)
														self.assertTrue(_is_whitespace("\n"						)						)
														self.assertTrue(_is_whitespace("\u00A0"						)						)

														self.assertFalse(_is_whitespace("A"						)						)
														self.assertFalse(_is_whitespace("-"						)						)






							def  A_     ( self     :					List[Any]						)				->			List[str]:





														"""simple docstring"""
														self.assertTrue(_is_control("\u0005"						)						)

														self.assertFalse(_is_control("A"						)						)
														self.assertFalse(_is_control(" "						)						)
														self.assertFalse(_is_control("\t"						)						)
														self.assertFalse(_is_control("\r"						)						)






							def  A_     ( self     :					Optional[int]						)				->			Optional[int]:





														"""simple docstring"""
														self.assertTrue(_is_punctuation("-"						)						)
														self.assertTrue(_is_punctuation("$"						)						)
														self.assertTrue(_is_punctuation("`"						)						)
														self.assertTrue(_is_punctuation("."						)						)

														self.assertFalse(_is_punctuation("A"						)						)
														self.assertFalse(_is_punctuation(" "						)						)






							@slow
							def  A_     ( self     :					Optional[Any]						)				->			List[Any]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      str     =						self.tokenizer_class.from_pretrained("microsoft/prophetnet-large-uncased"						)

														SCREAMING_SNAKE_CASE__       :      Tuple     =						tokenizer.encode("sequence builders",			add_special_tokens=_UpperCAmelCase						)
														SCREAMING_SNAKE_CASE__       :      List[str]     =						tokenizer.encode("multi-sequence build",			add_special_tokens=_UpperCAmelCase						)

														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase						)
														SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase,			_UpperCAmelCase						)

														assert encoded_sentence == text + [1_0_2]
														assert encoded_pair == text + [1_0_2] + text_a + [1_0_2]













import json
import os
from typing import Optional

import numpy as np

from ...feature_extraction_utils import BatchFeature
from ...processing_utils import ProcessorMixin
from ...utils import logging
from ...utils.hub import get_file_from_repo
from ..auto import AutoTokenizer


_lowerCamelCase :    Optional[int]  							= logging.get_logger(__name__)


class 						lowerCamelCase       (__lowerCamelCase      ):







							"""simple docstring"""




							UpperCAmelCase_      						=   "AutoTokenizer"
							UpperCAmelCase_      						=   ["tokenizer"]

							UpperCAmelCase_      						=   {
							    "semantic_prompt": 1,
							    "coarse_prompt": 2,
							    "fine_prompt": 2,
							}






							def __init__( self     :					Union[str, Any],			_UpperCAmelCase     :					Optional[int],			_UpperCAmelCase     :					Union[str, Any]=None						)				->			Union[str, Any]:





														"""simple docstring"""
														super().__init__(_UpperCAmelCase						)

														SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						speaker_embeddings






							@classmethod
							def  A_     ( cls     :					Any,			_UpperCAmelCase     :					List[str],			_UpperCAmelCase     :					Dict="speaker_embeddings_path.json",			**_UpperCAmelCase     :					Optional[Any]						)				->			List[Any]:





														"""simple docstring"""

														if speaker_embeddings_dict_path is not None:
																					SCREAMING_SNAKE_CASE__       :      Any     =						get_file_from_repo(
																					    _UpperCAmelCase,			_UpperCAmelCase,			subfolder=kwargs.pop("subfolder",			_UpperCAmelCase						),			cache_dir=kwargs.pop("cache_dir",			_UpperCAmelCase						),			force_download=kwargs.pop("force_download",			_UpperCAmelCase						),			proxies=kwargs.pop("proxies",			_UpperCAmelCase						),			resume_download=kwargs.pop("resume_download",			_UpperCAmelCase						),			local_files_only=kwargs.pop("local_files_only",			_UpperCAmelCase						),			use_auth_token=kwargs.pop("use_auth_token",			_UpperCAmelCase						),			revision=kwargs.pop("revision",			_UpperCAmelCase						),			)
																					if speaker_embeddings_path is None:
																												logger.warning(
																												    F'''`{os.path.join(_UpperCAmelCase,			_UpperCAmelCase						)}` does not exists
                    , no preloaded speaker embeddings will be used - Make sure to provide a correct path to the json
                    dictionnary if wanted, otherwise set `speaker_embeddings_dict_path=None`.'''						)
																												SCREAMING_SNAKE_CASE__       :      Dict     =						None
																					else:
																												with open(_UpperCAmelCase						) as speaker_embeddings_json:
																																			SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						json.load(_UpperCAmelCase						)
														else:
																					SCREAMING_SNAKE_CASE__       :      List[Any]     =						None

														SCREAMING_SNAKE_CASE__       :      List[str]     =						AutoTokenizer.from_pretrained(_UpperCAmelCase,			**_UpperCAmelCase						)

														return cls(tokenizer=_UpperCAmelCase,			speaker_embeddings=_UpperCAmelCase						)






							def  A_     ( self     :					str,			_UpperCAmelCase     :					Optional[int],			_UpperCAmelCase     :					List[str]="speaker_embeddings_path.json",			_UpperCAmelCase     :					Optional[Any]="speaker_embeddings",			_UpperCAmelCase     :					bool = False,			**_UpperCAmelCase     :					List[str],			)				->			Union[str, Any]:





														"""simple docstring"""
														if self.speaker_embeddings is not None:
																					os.makedirs(os.path.join(_UpperCAmelCase,			_UpperCAmelCase,			"v2"						),			exist_ok=_UpperCAmelCase						)

																					SCREAMING_SNAKE_CASE__       :      Optional[int]     =						{}

																					SCREAMING_SNAKE_CASE__       :      Any     =						save_directory

																					for prompt_key in self.speaker_embeddings:
																												if prompt_key != "repo_or_path":
																																			SCREAMING_SNAKE_CASE__       :      List[Any]     =						self._load_voice_preset(_UpperCAmelCase						)

																																			SCREAMING_SNAKE_CASE__       :      int     =						{}
																																			for key in self.speaker_embeddings[prompt_key]:
																																										np.save(
																																										    os.path.join(
																																										        embeddings_dict["repo_or_path"],			_UpperCAmelCase,			F'''{prompt_key}_{key}'''						),			voice_preset[key],			allow_pickle=_UpperCAmelCase,			)
																																										SCREAMING_SNAKE_CASE__       :      List[str]     =						os.path.join(_UpperCAmelCase,			F'''{prompt_key}_{key}.npy'''						)

																																			SCREAMING_SNAKE_CASE__       :      Optional[int]     =						tmp_dict

																					with open(os.path.join(_UpperCAmelCase,			_UpperCAmelCase						),			"w"						) as fp:
																												json.dump(_UpperCAmelCase,			_UpperCAmelCase						)

														super().save_pretrained(_UpperCAmelCase,			_UpperCAmelCase,			**_UpperCAmelCase						)






							def  A_     ( self     :					List[Any],			_UpperCAmelCase     :					str = None,			**_UpperCAmelCase     :					List[Any]						)				->			List[str]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      str     =						self.speaker_embeddings[voice_preset]

														SCREAMING_SNAKE_CASE__       :      Optional[int]     =						{}
														for key in ["semantic_prompt", "coarse_prompt", "fine_prompt"]:
																					if key not in voice_preset_paths:
																												raise ValueError(
																												    F'''Voice preset unrecognized, missing {key} as a key in self.speaker_embeddings[{voice_preset}].'''						)

																					SCREAMING_SNAKE_CASE__       :      List[Any]     =						get_file_from_repo(
																					    self.speaker_embeddings.get("repo_or_path",			"/"						),			voice_preset_paths[key],			subfolder=kwargs.pop("subfolder",			_UpperCAmelCase						),			cache_dir=kwargs.pop("cache_dir",			_UpperCAmelCase						),			force_download=kwargs.pop("force_download",			_UpperCAmelCase						),			proxies=kwargs.pop("proxies",			_UpperCAmelCase						),			resume_download=kwargs.pop("resume_download",			_UpperCAmelCase						),			local_files_only=kwargs.pop("local_files_only",			_UpperCAmelCase						),			use_auth_token=kwargs.pop("use_auth_token",			_UpperCAmelCase						),			revision=kwargs.pop("revision",			_UpperCAmelCase						),			)
																					if path is None:
																												raise ValueError(
																												    F'''`{os.path.join(self.speaker_embeddings.get("repo_or_path",			"/"						),			voice_preset_paths[key]						)}` does not exists
                    , no preloaded voice preset will be used - Make sure to provide correct paths to the {voice_preset}
                    embeddings.'''						)

																					SCREAMING_SNAKE_CASE__       :      int     =						np.load(_UpperCAmelCase						)

														return voice_preset_dict






							def  A_     ( self     :					int,			_UpperCAmelCase     :					Optional[dict] = None						)				->			Optional[int]:





														"""simple docstring"""
														for key in ["semantic_prompt", "coarse_prompt", "fine_prompt"]:
																					if key not in voice_preset:
																												raise ValueError(F'''Voice preset unrecognized, missing {key} as a key.'''						)

																					if not isinstance(voice_preset[key],			np.ndarray						):
																												raise ValueError(F'''{key} voice preset must be a {str(self.preset_shape[key]						)}D ndarray.'''						)

																					if len(voice_preset[key].shape						) != self.preset_shape[key]:
																												raise ValueError(F'''{key} voice preset must be a {str(self.preset_shape[key]						)}D ndarray.'''						)






							def __call__( self     :					List[Any],			_UpperCAmelCase     :					Optional[Any]=None,			_UpperCAmelCase     :					Union[str, Any]=None,			_UpperCAmelCase     :					Optional[int]="pt",			_UpperCAmelCase     :					List[str]=2_5_6,			_UpperCAmelCase     :					int=False,			_UpperCAmelCase     :					Optional[int]=True,			_UpperCAmelCase     :					Any=False,			**_UpperCAmelCase     :					List[str],			)				->			List[Any]:





														"""simple docstring"""
														if voice_preset is not None and not isinstance(_UpperCAmelCase,			_UpperCAmelCase						):
																					if (
																					    isinstance(_UpperCAmelCase,			_UpperCAmelCase						)
																					    and self.speaker_embeddings is not None
																					    and voice_preset in self.speaker_embeddings
																					):
																												SCREAMING_SNAKE_CASE__       :      List[str]     =						self._load_voice_preset(_UpperCAmelCase						)

																					else:
																												if isinstance(_UpperCAmelCase,			_UpperCAmelCase						) and not voice_preset.endswith(".npz"						):
																																			SCREAMING_SNAKE_CASE__       :      Optional[int]     =						voice_preset + ".npz"

																												SCREAMING_SNAKE_CASE__       :      List[str]     =						np.load(_UpperCAmelCase						)

														if voice_preset is not None:
																					self._validate_voice_preset_dict(_UpperCAmelCase,			**_UpperCAmelCase						)
																					SCREAMING_SNAKE_CASE__       :      Any     =						BatchFeature(data=_UpperCAmelCase,			tensor_type=_UpperCAmelCase						)

														SCREAMING_SNAKE_CASE__       :      Dict     =						self.tokenizer(
														    _UpperCAmelCase,			return_tensors=_UpperCAmelCase,			padding="max_length",			max_length=_UpperCAmelCase,			return_attention_mask=_UpperCAmelCase,			return_token_type_ids=_UpperCAmelCase,			add_special_tokens=_UpperCAmelCase,			**_UpperCAmelCase,			)

														if voice_preset is not None:
																					SCREAMING_SNAKE_CASE__       :      str     =						voice_preset

														return encoded_text






