
import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
    from PIL import Image

    from transformers import AlignProcessor, EfficientNetImageProcessor




@require_vision
class      __A(    unittest.TestCase			):

  """simple docstring"""







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    tempfile.mkdtemp()

       UpperCamelCase__				=    [
           """[UNK]""",
           """[CLS]""",
           """[SEP]""",
           """[PAD]""",
           """[MASK]""",
           """want""",
           """##want""",
           """##ed""",
           """wa""",
           """un""",
           """runn""",
           """##ing""",
           """,""",
           """low""",
           """lowest""",
       ]
       UpperCamelCase__				=    os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES["""vocab_file"""]						)
       with open(self.vocab_file	,      """w"""	,      encoding="""utf-8"""						) as vocab_writer:
            vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)

       UpperCamelCase__				=    {
           """do_resize""": True,
           """size""": 20,
           """do_center_crop""": True,
           """crop_size""": 18,
           """do_normalize""": True,
           """image_mean""": [0.4814_5466, 0.457_8275, 0.4082_1073],
           """image_std""": [0.2686_2954, 0.2613_0258, 0.2757_7711],
       }
       UpperCamelCase__				=    os.path.join(self.tmpdirname	,      SCREAMING_SNAKE_CASE_						)
       with open(self.image_processor_file	,      """w"""	,      encoding="""utf-8"""						) as fp:
            json.dump(SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_						)







  def 	UpperCAmelCase_ (self	,      **SCREAMING_SNAKE_CASE_						):
       return BertTokenizer.from_pretrained(self.tmpdirname	,      **SCREAMING_SNAKE_CASE_						)







  def 	UpperCAmelCase_ (self	,      **SCREAMING_SNAKE_CASE_						):
       return BertTokenizerFast.from_pretrained(self.tmpdirname	,      **SCREAMING_SNAKE_CASE_						)







  def 	UpperCAmelCase_ (self	,      **SCREAMING_SNAKE_CASE_						):
       return EfficientNetImageProcessor.from_pretrained(self.tmpdirname	,      **SCREAMING_SNAKE_CASE_						)







  def 	UpperCAmelCase_ (self						):
       shutil.rmtree(self.tmpdirname						)







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    [np.random.randint(2_55	,      size=(3, 30, 4_00)	,      dtype=np.uinta						)]
       UpperCamelCase__				=    [Image.fromarray(np.moveaxis(SCREAMING_SNAKE_CASE_	,      0	,      -1						)						) for x in image_inputs]
       return image_inputs







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    self.get_tokenizer()
       UpperCamelCase__				=    self.get_rust_tokenizer()
       UpperCamelCase__				=    self.get_image_processor()

       UpperCamelCase__				=    AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_	,      image_processor=SCREAMING_SNAKE_CASE_						)
       processor_slow.save_pretrained(self.tmpdirname						)
       UpperCamelCase__				=    AlignProcessor.from_pretrained(self.tmpdirname	,      use_fast=SCREAMING_SNAKE_CASE_						)

       UpperCamelCase__				=    AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_	,      image_processor=SCREAMING_SNAKE_CASE_						)
       processor_fast.save_pretrained(self.tmpdirname						)
       UpperCamelCase__				=    AlignProcessor.from_pretrained(self.tmpdirname						)

       self.assertEqual(processor_slow.tokenizer.get_vocab()	,      tokenizer_slow.get_vocab()						)
       self.assertEqual(processor_fast.tokenizer.get_vocab()	,      tokenizer_fast.get_vocab()						)
       self.assertEqual(tokenizer_slow.get_vocab()	,      tokenizer_fast.get_vocab()						)
       self.assertIsInstance(processor_slow.tokenizer	,      SCREAMING_SNAKE_CASE_						)
       self.assertIsInstance(processor_fast.tokenizer	,      SCREAMING_SNAKE_CASE_						)

       self.assertEqual(processor_slow.image_processor.to_json_string()	,      image_processor.to_json_string()						)
       self.assertEqual(processor_fast.image_processor.to_json_string()	,      image_processor.to_json_string()						)
       self.assertIsInstance(processor_slow.image_processor	,      SCREAMING_SNAKE_CASE_						)
       self.assertIsInstance(processor_fast.image_processor	,      SCREAMING_SNAKE_CASE_						)







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    AlignProcessor(tokenizer=self.get_tokenizer()	,      image_processor=self.get_image_processor()						)
       processor.save_pretrained(self.tmpdirname						)

       UpperCamelCase__				=    self.get_tokenizer(bos_token="""(BOS)"""	,      eos_token="""(EOS)"""						)
       UpperCamelCase__				=    self.get_image_processor(do_normalize=SCREAMING_SNAKE_CASE_	,      padding_value=1.0						)

       UpperCamelCase__				=    AlignProcessor.from_pretrained(
           self.tmpdirname	,      bos_token="""(BOS)"""	,      eos_token="""(EOS)"""	,      do_normalize=SCREAMING_SNAKE_CASE_	,      padding_value=1.0						)

       self.assertEqual(processor.tokenizer.get_vocab()	,      tokenizer_add_kwargs.get_vocab()						)
       self.assertIsInstance(processor.tokenizer	,      SCREAMING_SNAKE_CASE_						)

       self.assertEqual(processor.image_processor.to_json_string()	,      image_processor_add_kwargs.to_json_string()						)
       self.assertIsInstance(processor.image_processor	,      SCREAMING_SNAKE_CASE_						)







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    self.get_image_processor()
       UpperCamelCase__				=    self.get_tokenizer()

       UpperCamelCase__				=    AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_	,      image_processor=SCREAMING_SNAKE_CASE_						)

       UpperCamelCase__				=    self.prepare_image_inputs()

       UpperCamelCase__				=    image_processor(SCREAMING_SNAKE_CASE_	,      return_tensors="""np"""						)
       UpperCamelCase__				=    processor(images=SCREAMING_SNAKE_CASE_	,      return_tensors="""np"""						)

       for key in input_image_proc.keys():
            self.assertAlmostEqual(input_image_proc[key].sum()	,      input_processor[key].sum()	,      delta=1E-2						)







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    self.get_image_processor()
       UpperCamelCase__				=    self.get_tokenizer()

       UpperCamelCase__				=    AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_	,      image_processor=SCREAMING_SNAKE_CASE_						)

       UpperCamelCase__				=    """lower newer"""

       UpperCamelCase__				=    processor(text=SCREAMING_SNAKE_CASE_						)

       UpperCamelCase__				=    tokenizer(SCREAMING_SNAKE_CASE_	,      padding="""max_length"""	,      max_length=64						)

       for key in encoded_tok.keys():
            self.assertListEqual(encoded_tok[key]	,      encoded_processor[key]						)







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    self.get_image_processor()
       UpperCamelCase__				=    self.get_tokenizer()

       UpperCamelCase__				=    AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_	,      image_processor=SCREAMING_SNAKE_CASE_						)

       UpperCamelCase__				=    """lower newer"""
       UpperCamelCase__				=    self.prepare_image_inputs()

       UpperCamelCase__				=    processor(text=SCREAMING_SNAKE_CASE_	,      images=SCREAMING_SNAKE_CASE_						)

       self.assertListEqual(list(inputs.keys()						)	,      ["""input_ids""", """token_type_ids""", """attention_mask""", """pixel_values"""]						)

       # test if it raises when no input is passed
       with pytest.raises(SCREAMING_SNAKE_CASE_						):
            processor()







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    self.get_image_processor()
       UpperCamelCase__				=    self.get_tokenizer()

       UpperCamelCase__				=    AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_	,      image_processor=SCREAMING_SNAKE_CASE_						)

       UpperCamelCase__				=    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

       UpperCamelCase__				=    processor.batch_decode(SCREAMING_SNAKE_CASE_						)
       UpperCamelCase__				=    tokenizer.batch_decode(SCREAMING_SNAKE_CASE_						)

       self.assertListEqual(SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_						)







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    self.get_image_processor()
       UpperCamelCase__				=    self.get_tokenizer()

       UpperCamelCase__				=    AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_	,      image_processor=SCREAMING_SNAKE_CASE_						)

       UpperCamelCase__				=    """lower newer"""
       UpperCamelCase__				=    self.prepare_image_inputs()

       UpperCamelCase__				=    processor(text=SCREAMING_SNAKE_CASE_	,      images=SCREAMING_SNAKE_CASE_						)

       self.assertListEqual(list(inputs.keys()						)	,      processor.model_input_names						)







import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows






def 						__magic_name__			(   __a      :	Any						):  # picklable for multiprocessing



     '''simple docstring'''




     return i + 1






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def 						__magic_name__			(   ):



     '''simple docstring'''




     with parallel_backend("""spark"""						):
          assert ParallelBackendConfig.backend_name == "spark"

     UpperCamelCase__				=    [1, 2, 3]
     with pytest.raises(__a						):
          with parallel_backend("""unsupported backend"""						):
               map_nested(__a	,		__a	,		num_proc=2						)

     with pytest.raises(__a						):
          with parallel_backend("""unsupported backend"""						):
               map_nested(__a	,		__a	,		num_proc=-1						)






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize("""num_proc"""	,		[2, -1]						)
def 						__magic_name__			(   __a      :	Optional[int]						):



     '''simple docstring'''




     UpperCamelCase__				=    [1, 2]
     UpperCamelCase__				=    {"""a""": 1, """b""": 2}
     UpperCamelCase__				=    {"""a""": [1, 2], """b""": [3, 4]}
     UpperCamelCase__				=    {"""a""": {"""1""": 1}, """b""": 2}
     UpperCamelCase__				=    {"""a""": 1, """b""": 2, """c""": 3, """d""": 4}
     UpperCamelCase__				=    [2, 3]
     UpperCamelCase__				=    {"""a""": 2, """b""": 3}
     UpperCamelCase__				=    {"""a""": [2, 3], """b""": [4, 5]}
     UpperCamelCase__				=    {"""a""": {"""1""": 2}, """b""": 3}
     UpperCamelCase__				=    {"""a""": 2, """b""": 3, """c""": 4, """d""": 5}

     with parallel_backend("""spark"""						):
          assert map_nested(__a	,		__a	,		num_proc=__a						) == expected_map_nested_sa
          assert map_nested(__a	,		__a	,		num_proc=__a						) == expected_map_nested_sa
          assert map_nested(__a	,		__a	,		num_proc=__a						) == expected_map_nested_sa
          assert map_nested(__a	,		__a	,		num_proc=__a						) == expected_map_nested_sa
          assert map_nested(__a	,		__a	,		num_proc=__a						) == expected_map_nested_sa





