


"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


__SCREAMING_SNAKE_CASE							:      int						=      logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							:      int						=      {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE							:      List[Any]						=      {
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

__SCREAMING_SNAKE_CASE							:      Optional[int]						=      {
    'google/realm-cc-news-pretrained-embedder': 512,
    'google/realm-cc-news-pretrained-encoder': 512,
    'google/realm-cc-news-pretrained-scorer': 512,
    'google/realm-cc-news-pretrained-openqa': 512,
    'google/realm-orqa-nq-openqa': 512,
    'google/realm-orqa-nq-reader': 512,
    'google/realm-orqa-wq-openqa': 512,
    'google/realm-orqa-wq-reader': 512,
}

__SCREAMING_SNAKE_CASE							:      Optional[int]						=      {
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}





class        __A       (snake_case__):





							'''simple docstring'''

							__lowercase:						Optional[Any]							 =      VOCAB_FILES_NAMES
							__lowercase:						List[str]							 =      PRETRAINED_VOCAB_FILES_MAP
							__lowercase:						Union[str, Any]							 =      PRETRAINED_INIT_CONFIGURATION
							__lowercase:						Optional[Any]							 =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase:						Optional[Any]							 =      RealmTokenizer



							def __init__(			self							: int						,    UpperCAmelCase_							: Any=None						,    UpperCAmelCase_							: Union[str, Any]=None						,    UpperCAmelCase_							: str=True						,    UpperCAmelCase_							: Any="[UNK]"						,    UpperCAmelCase_							: Optional[int]="[SEP]"						,    UpperCAmelCase_							: Optional[int]="[PAD]"						,    UpperCAmelCase_							: List[str]="[CLS]"						,    UpperCAmelCase_							: Tuple="[MASK]"						,    UpperCAmelCase_							: Optional[int]=True						,    UpperCAmelCase_							: Tuple=None						,    **UpperCAmelCase_							: Optional[Any]						,    )   ->int:



														"""simple docstring"""



														super().__init__(
														    UpperCAmelCase_						,    tokenizer_file=UpperCAmelCase_						,    do_lower_case=UpperCAmelCase_						,    unk_token=UpperCAmelCase_						,    sep_token=UpperCAmelCase_						,    pad_token=UpperCAmelCase_						,    cls_token=UpperCAmelCase_						,    mask_token=UpperCAmelCase_						,    tokenize_chinese_chars=UpperCAmelCase_						,    strip_accents=UpperCAmelCase_						,    **UpperCAmelCase_						,    )

														snake_case_          =						json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
														if (
														    normalizer_state.get("""lowercase"""						,    UpperCAmelCase_    ) != do_lower_case
														    or normalizer_state.get("""strip_accents"""						,    UpperCAmelCase_    ) != strip_accents
														    or normalizer_state.get("""handle_chinese_chars"""						,    UpperCAmelCase_    ) != tokenize_chinese_chars
														):
																					snake_case_          =						getattr(UpperCAmelCase_						,    normalizer_state.pop("""type"""    )    )
																					snake_case_          =						do_lower_case
																					snake_case_          =						strip_accents
																					snake_case_          =						tokenize_chinese_chars
																					snake_case_          =						normalizer_class(**UpperCAmelCase_    )

														snake_case_          =						do_lower_case



							def 						lowerCAmelCase    (			self							: Union[str, Any]						,    UpperCAmelCase_							: int						,    **UpperCAmelCase_							: List[str]    )   ->Union[str, Any]:



														"""simple docstring"""



														snake_case_          =						PaddingStrategy.MAX_LENGTH

														snake_case_          =						text
														snake_case_          =						kwargs.pop("""text_pair"""						,    UpperCAmelCase_    )
														snake_case_          =						kwargs.pop("""return_tensors"""						,    UpperCAmelCase_    )

														snake_case_          =						{
														    """input_ids""": [],
														    """attention_mask""": [],
														    """token_type_ids""": [],
														}

														for idx, candidate_text in enumerate(UpperCAmelCase_    ):
																					if batch_text_pair is not None:
																												snake_case_          =						batch_text_pair[idx]
																					else:
																												snake_case_          =						None

																					snake_case_          =						super().__call__(UpperCAmelCase_						,    UpperCAmelCase_						,    return_tensors=UpperCAmelCase_						,    **UpperCAmelCase_    )

																					snake_case_          =						encoded_candidates.get("""input_ids"""    )
																					snake_case_          =						encoded_candidates.get("""attention_mask"""    )
																					snake_case_          =						encoded_candidates.get("""token_type_ids"""    )

																					if encoded_input_ids is not None:
																												output_data["input_ids"].append(UpperCAmelCase_    )
																					if encoded_attention_mask is not None:
																												output_data["attention_mask"].append(UpperCAmelCase_    )
																					if encoded_token_type_ids is not None:
																												output_data["token_type_ids"].append(UpperCAmelCase_    )

														snake_case_          =						{key: item for key, item in output_data.items() if len(UpperCAmelCase_    ) != 0}

														return BatchEncoding(UpperCAmelCase_						,    tensor_type=UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: Tuple						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: Dict=None    )   ->int:



														"""simple docstring"""



														snake_case_          =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output



							def 						lowerCAmelCase    (			self							: str						,    UpperCAmelCase_							: List[int]						,    UpperCAmelCase_							: Optional[List[int]] = None    )   ->List[int]:



														"""simple docstring"""



														snake_case_          =						[self.sep_token_id]
														snake_case_          =						[self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep    ) * [0]
														return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]



							def 						lowerCAmelCase    (			self							: Dict						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: Optional[str] = None    )   ->Tuple[str]:



														"""simple docstring"""



														snake_case_          =						self._tokenizer.model.save(UpperCAmelCase_						,    name=UpperCAmelCase_    )
														return tuple(UpperCAmelCase_    )







"""simple docstring"""
import shutil
import tempfile
import unittest

from transformers import ClapFeatureExtractor, ClapProcessor, RobertaTokenizer, RobertaTokenizerFast
from transformers.testing_utils import require_sentencepiece, require_torchaudio

from .test_feature_extraction_clap import floats_list





@require_torchaudio
@require_sentencepiece
class        __A       (unittest.TestCase):





							'''simple docstring'''

							def 						lowerCAmelCase    (			self							: Dict    )   ->List[Any]:



														"""simple docstring"""



														snake_case_          =						"""laion/clap-htsat-unfused"""
														snake_case_          =						tempfile.mkdtemp()



							def 						lowerCAmelCase    (			self							: List[str]						,    **UpperCAmelCase_							: Tuple    )   ->str:



														"""simple docstring"""



														return RobertaTokenizer.from_pretrained(self.checkpoint						,    **UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: Optional[Any]						,    **UpperCAmelCase_							: Any    )   ->Optional[Any]:



														"""simple docstring"""



														return ClapFeatureExtractor.from_pretrained(self.checkpoint						,    **UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: str    )   ->Any:



														"""simple docstring"""



														shutil.rmtree(self.tmpdirname    )



							def 						lowerCAmelCase    (			self							: Tuple    )   ->Optional[int]:



														"""simple docstring"""



														snake_case_          =						self.get_tokenizer()
														snake_case_          =						self.get_feature_extractor()

														snake_case_          =						ClapProcessor(tokenizer=UpperCAmelCase_						,    feature_extractor=UpperCAmelCase_    )

														processor.save_pretrained(self.tmpdirname    )
														snake_case_          =						ClapProcessor.from_pretrained(self.tmpdirname    )

														self.assertEqual(processor.tokenizer.get_vocab()						,    tokenizer.get_vocab()    )
														self.assertIsInstance(processor.tokenizer						,    UpperCAmelCase_    )

														self.assertEqual(processor.feature_extractor.to_json_string()						,    feature_extractor.to_json_string()    )
														self.assertIsInstance(processor.feature_extractor						,    UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: int    )   ->List[str]:



														"""simple docstring"""



														snake_case_          =						ClapProcessor(tokenizer=self.get_tokenizer()						,    feature_extractor=self.get_feature_extractor()    )
														processor.save_pretrained(self.tmpdirname    )

														snake_case_          =						self.get_tokenizer(bos_token="""(BOS)"""						,    eos_token="""(EOS)"""    )
														snake_case_          =						self.get_feature_extractor(do_normalize=UpperCAmelCase_						,    padding_value=1.0    )

														snake_case_          =						ClapProcessor.from_pretrained(
														    self.tmpdirname						,    bos_token="""(BOS)"""						,    eos_token="""(EOS)"""						,    do_normalize=UpperCAmelCase_						,    padding_value=1.0    )

														self.assertEqual(processor.tokenizer.get_vocab()						,    tokenizer_add_kwargs.get_vocab()    )
														self.assertIsInstance(processor.tokenizer						,    UpperCAmelCase_    )

														self.assertEqual(processor.feature_extractor.to_json_string()						,    feature_extractor_add_kwargs.to_json_string()    )
														self.assertIsInstance(processor.feature_extractor						,    UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: str    )   ->Dict:



														"""simple docstring"""



														snake_case_          =						self.get_feature_extractor()
														snake_case_          =						self.get_tokenizer()

														snake_case_          =						ClapProcessor(tokenizer=UpperCAmelCase_						,    feature_extractor=UpperCAmelCase_    )

														snake_case_          =						floats_list((3, 1_000)    )

														snake_case_          =						feature_extractor(UpperCAmelCase_						,    return_tensors="""np"""    )
														snake_case_          =						processor(audios=UpperCAmelCase_						,    return_tensors="""np"""    )

														for key in input_feat_extract.keys():
																					self.assertAlmostEqual(input_feat_extract[key].sum()						,    input_processor[key].sum()						,    delta=1E-2    )



							def 						lowerCAmelCase    (			self							: Union[str, Any]    )   ->Optional[int]:



														"""simple docstring"""



														snake_case_          =						self.get_feature_extractor()
														snake_case_          =						self.get_tokenizer()

														snake_case_          =						ClapProcessor(tokenizer=UpperCAmelCase_						,    feature_extractor=UpperCAmelCase_    )

														snake_case_          =						"""This is a test string"""

														snake_case_          =						processor(text=UpperCAmelCase_    )

														snake_case_          =						tokenizer(UpperCAmelCase_    )

														for key in encoded_tok.keys():
																					self.assertListEqual(encoded_tok[key]						,    encoded_processor[key]    )



							def 						lowerCAmelCase    (			self							: Optional[int]    )   ->Tuple:



														"""simple docstring"""



														snake_case_          =						self.get_feature_extractor()
														snake_case_          =						self.get_tokenizer()

														snake_case_          =						ClapProcessor(tokenizer=UpperCAmelCase_						,    feature_extractor=UpperCAmelCase_    )

														snake_case_          =						[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

														snake_case_          =						processor.batch_decode(UpperCAmelCase_    )
														snake_case_          =						tokenizer.batch_decode(UpperCAmelCase_    )

														self.assertListEqual(UpperCAmelCase_						,    UpperCAmelCase_    )



							def 						lowerCAmelCase    (			self							: List[str]    )   ->Union[str, Any]:



														"""simple docstring"""



														snake_case_          =						self.get_feature_extractor()
														snake_case_          =						self.get_tokenizer()

														snake_case_          =						ClapProcessor(tokenizer=UpperCAmelCase_						,    feature_extractor=UpperCAmelCase_    )

														self.assertListEqual(
														    processor.model_input_names[2:]						,    feature_extractor.model_input_names						,    msg="""`processor` and `feature_extractor` model input names do not match"""						,    )



