




"""simple docstring"""



import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin




class      UpperCAmelCase_  (						UpperCamelCase_,       unittest.TestCase							):
    lowercase__						 =				ProphetNetTokenizer
    lowercase__						 =				False




    def 							__magic_name__							(   self						:	Tuple				)				->							Union[str, Any]:




     '''simple docstring'''

     super().setUp()

     A__	=    [
         """[UNK]""",
         """[CLS]""",
         """[SEP]""",
         """[PAD]""",
         """[MASK]""",
         """want""",
         """##want""",
         """##ed""",
         """wa""",
         """un""",
         """runn""",
         """##ing""",
         """,""",
         """low""",
         """lowest""",
     ]
     A__	=    os.path.join(self.tmpdirname     ,			VOCAB_FILES_NAMES["vocab_file"]				)
     with open(self.vocab_file     ,			"w"     ,			encoding="utf-8"				) as vocab_writer:
      vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)




    def 							__magic_name__							(   self						:	str     ,			snake_case_						:	Union[str, Any]				)				->							Union[str, Any]:




     '''simple docstring'''

     A__	=    """UNwant\u00E9d,running"""
     A__	=    """unwanted, running"""
     return input_text, output_text




    def 							__magic_name__							(   self						:	Union[str, Any]				)				->							Dict:




     '''simple docstring'''

     A__	=    self.tokenizer_class(self.vocab_file				)

     A__	=    tokenizer.tokenize("UNwant\u00E9d,running"				)
     self.assertListEqual(_a     ,			["un", "##want", "##ed", ",", "runn", "##ing"]				)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_a				)     ,			[9, 6, 7, 12, 10, 11]				)




    def 							__magic_name__							(   self						:	List[str]				)				->							Tuple:




     '''simple docstring'''

     A__	=    BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)     ,			["ah", "\u535A", "\u63A8", "zz"]				)




    def 							__magic_name__							(   self						:	Any				)				->							List[Any]:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=_a				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)     ,			["hello", "!", "how", "are", "you", "?"]				)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,			["hello"]				)




    def 							__magic_name__							(   self						:	List[str]				)				->							Tuple:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=_a     ,			strip_accents=_a				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,			["hällo", "!", "how", "are", "you", "?"]				)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,			["h\u00E9llo"]				)




    def 							__magic_name__							(   self						:	Dict				)				->							Optional[int]:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=_a     ,			strip_accents=_a				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,			["hallo", "!", "how", "are", "you", "?"]				)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,			["hello"]				)




    def 							__magic_name__							(   self						:	Tuple				)				->							Dict:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=_a				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,			["hallo", "!", "how", "are", "you", "?"]				)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,			["hello"]				)




    def 							__magic_name__							(   self						:	int				)				->							Optional[int]:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=_a				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)     ,			["HeLLo", "!", "how", "Are", "yoU", "?"]				)




    def 							__magic_name__							(   self						:	List[Any]				)				->							str:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=_a     ,			strip_accents=_a				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,			["HäLLo", "!", "how", "Are", "yoU", "?"]				)




    def 							__magic_name__							(   self						:	Optional[Any]				)				->							Tuple:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=_a     ,			strip_accents=_a				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,			["HaLLo", "!", "how", "Are", "yoU", "?"]				)




    def 							__magic_name__							(   self						:	List[Any]				)				->							Optional[int]:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=_a     ,			never_split=["[UNK]"]				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)     ,			["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)




    def 							__magic_name__							(   self						:	List[Any]				)				->							List[Any]:




     '''simple docstring'''

     A__	=    ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

     A__	=    {}
     for i, token in enumerate(_a				):
      A__	=    i
     A__	=    WordpieceTokenizer(vocab=_a     ,			unk_token="[UNK]"				)

     self.assertListEqual(tokenizer.tokenize(""				)     ,			[]				)

     self.assertListEqual(tokenizer.tokenize("unwanted running"				)     ,			["un", "##want", "##ed", "runn", "##ing"]				)

     self.assertListEqual(tokenizer.tokenize("unwantedX running"				)     ,			["[UNK]", "runn", "##ing"]				)




    @require_torch
    def 							__magic_name__							(   self						:	Tuple				)				->							str:




     '''simple docstring'''

     A__	=    self.tokenizer_class.from_pretrained("microsoft/prophetnet-large-uncased"				)

     A__	=    ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
     A__	=    [1_037, 2_146, 20_423, 2_005, 7_680, 7_849, 3_989, 1_012, 102]
     A__	=    tokenizer(_a     ,			padding=_a     ,			return_tensors="pt"				)
     self.assertIsInstance(_a     ,			_a				)
     A__	=    list(batch.input_ids.numpy()[0]				)
     self.assertListEqual(_a     ,			_a				)

     self.assertEqual((2, 9)     ,			batch.input_ids.shape				)
     self.assertEqual((2, 9)     ,			batch.attention_mask.shape				)




    def 							__magic_name__							(   self						:	Dict				)				->							Tuple:




     '''simple docstring'''

     self.assertTrue(_is_whitespace(" "				)				)
     self.assertTrue(_is_whitespace("\t"				)				)
     self.assertTrue(_is_whitespace("\r"				)				)
     self.assertTrue(_is_whitespace("\n"				)				)
     self.assertTrue(_is_whitespace("\u00A0"				)				)

     self.assertFalse(_is_whitespace("A"				)				)
     self.assertFalse(_is_whitespace("-"				)				)




    def 							__magic_name__							(   self						:	Any				)				->							int:




     '''simple docstring'''

     self.assertTrue(_is_control("\u0005"				)				)

     self.assertFalse(_is_control("A"				)				)
     self.assertFalse(_is_control(" "				)				)
     self.assertFalse(_is_control("\t"				)				)
     self.assertFalse(_is_control("\r"				)				)




    def 							__magic_name__							(   self						:	str				)				->							int:




     '''simple docstring'''

     self.assertTrue(_is_punctuation("-"				)				)
     self.assertTrue(_is_punctuation("$"				)				)
     self.assertTrue(_is_punctuation("`"				)				)
     self.assertTrue(_is_punctuation("."				)				)

     self.assertFalse(_is_punctuation("A"				)				)
     self.assertFalse(_is_punctuation(" "				)				)




    @slow
    def 							__magic_name__							(   self						:	List[str]				)				->							List[str]:




     '''simple docstring'''

     A__	=    self.tokenizer_class.from_pretrained("microsoft/prophetnet-large-uncased"				)

     A__	=    tokenizer.encode("sequence builders"     ,			add_special_tokens=_a				)
     A__	=    tokenizer.encode("multi-sequence build"     ,			add_special_tokens=_a				)

     A__	=    tokenizer.build_inputs_with_special_tokens(_a				)
     A__	=    tokenizer.build_inputs_with_special_tokens(_a     ,			_a				)

     assert encoded_sentence == text + [102]
     assert encoded_pair == text + [102] + text_a + [102]











"""simple docstring"""



import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import is_speech_available, is_vision_available
from transformers.testing_utils import require_torch


if is_vision_available():
      from transformers import TvltImageProcessor

if is_speech_available():
      from transformers import TvltFeatureExtractor

from transformers import TvltProcessor




@require_torch
class      UpperCAmelCase_  (						unittest.TestCase							):




    def 							__magic_name__							(   self						:	str				)				->							Dict:




     '''simple docstring'''

     A__	=    "ZinengTang/tvlt-base"
     A__	=    tempfile.mkdtemp()




    def 							__magic_name__							(   self						:	int     ,			**snake_case_						:	Dict				)				->							Union[str, Any]:




     '''simple docstring'''

     return TvltImageProcessor.from_pretrained(self.checkpoint     ,			**snake_case_				)




    def 							__magic_name__							(   self						:	Optional[int]     ,			**snake_case_						:	str				)				->							List[str]:




     '''simple docstring'''

     return TvltFeatureExtractor.from_pretrained(self.checkpoint     ,			**snake_case_				)




    def 							__magic_name__							(   self						:	Optional[int]				)				->							str:




     '''simple docstring'''

     shutil.rmtree(self.tmpdirname				)




    def 							__magic_name__							(   self						:	Tuple				)				->							Any:




     '''simple docstring'''

     A__	=    self.get_image_processor()
     A__	=    self.get_feature_extractor()

     A__	=    TvltProcessor(image_processor=snake_case_     ,			feature_extractor=snake_case_				)
     processor.save_pretrained(self.tmpdirname				)
     A__	=    TvltProcessor.from_pretrained(self.tmpdirname				)

     self.assertIsInstance(processor.feature_extractor     ,			snake_case_				)
     self.assertIsInstance(processor.image_processor     ,			snake_case_				)




    def 							__magic_name__							(   self						:	Optional[Any]				)				->							Union[str, Any]:




     '''simple docstring'''

     A__	=    self.get_image_processor()
     A__	=    self.get_feature_extractor()

     A__	=    TvltProcessor(image_processor=snake_case_     ,			feature_extractor=snake_case_				)

     A__	=    np.ones([12_000]				)

     A__	=    feature_extractor(snake_case_     ,			return_tensors="np"				)
     A__	=    processor(audio=snake_case_     ,			return_tensors="np"				)

     for key in audio_dict.keys():
      self.assertAlmostEqual(audio_dict[key].sum()     ,			input_processor[key].sum()     ,			delta=1e-2				)




    def 							__magic_name__							(   self						:	Tuple				)				->							Tuple:




     '''simple docstring'''

     A__	=    self.get_image_processor()
     A__	=    self.get_feature_extractor()

     A__	=    TvltProcessor(image_processor=snake_case_     ,			feature_extractor=snake_case_				)

     A__	=    np.ones([3, 224, 224]				)

     A__	=    image_processor(snake_case_     ,			return_tensors="np"				)
     A__	=    processor(images=snake_case_     ,			return_tensors="np"				)

     for key in image_dict.keys():
      self.assertAlmostEqual(image_dict[key].sum()     ,			input_processor[key].sum()     ,			delta=1e-2				)




    def 							__magic_name__							(   self						:	Tuple				)				->							Optional[int]:




     '''simple docstring'''

     A__	=    self.get_image_processor()
     A__	=    self.get_feature_extractor()

     A__	=    TvltProcessor(image_processor=snake_case_     ,			feature_extractor=snake_case_				)

     A__	=    np.ones([12_000]				)
     A__	=    np.ones([3, 224, 224]				)

     A__	=    processor(audio=snake_case_     ,			images=snake_case_				)

     self.assertListEqual(list(inputs.keys()				)     ,			["audio_values", "audio_mask", "pixel_values", "pixel_mask"]				)

     # test if it raises when no input is passed
     with pytest.raises(snake_case_				):
      processor()




    def 							__magic_name__							(   self						:	Optional[Any]				)				->							Tuple:




     '''simple docstring'''

     A__	=    self.get_image_processor()
     A__	=    self.get_feature_extractor()

     A__	=    TvltProcessor(image_processor=snake_case_     ,			feature_extractor=snake_case_				)

     self.assertListEqual(
         processor.model_input_names     ,			image_processor.model_input_names + feature_extractor.model_input_names     ,			msg="`processor` and `image_processor`+`feature_extractor` model input names do not match"     ,			)





