

import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam

class 			a					(		datasets.BeamBasedBuilder			):


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										return datasets.DatasetInfo(
										    features=datasets.Features({"content": datasets.Value("string"		)}		)  ,   supervised_keys=A_  ,   )


						def     _UpperCAmelCase			(    self  ,   A_  ,   A_		):







										'''simple docstring'''


										return [datasets.SplitGenerator(name=datasets.Split.TRAIN  ,   gen_kwargs={"examples": get_test_dummy_examples()}		)]





						def     _UpperCAmelCase			(    self  ,   A_  ,   A_		):







										'''simple docstring'''


										import apache_beam as beam

										return pipeline | "Load Examples" >> beam.Create(A_		)







class 			a					(		datasets.BeamBasedBuilder			):


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										return datasets.DatasetInfo(
										    features=datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"		)}		)}		)  ,   supervised_keys=A_  ,   )


						def     _UpperCAmelCase			(    self  ,   A_  ,   A_		):







										'''simple docstring'''


										return [
										    datasets.SplitGenerator(name=datasets.Split.TRAIN  ,   gen_kwargs={"examples": get_test_nested_examples()}		)
										]





						def     _UpperCAmelCase			(    self  ,   A_  ,   A_		):







										'''simple docstring'''


										import apache_beam as beam

										return pipeline | "Load Examples" >> beam.Create(A_		)







def 							__SCREAMING_SNAKE_CASE							(			)				->    List[str]:
				return [(i, {"content": content}) for i, content in enumerate(["foo", "bar", "foobar"]	)]

def 							__SCREAMING_SNAKE_CASE							(			)				->    List[str]:
				return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["foo", "bar", "foobar"]	)]





class 			a					(		UpperCAmelCase			):


						@require_beam
						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						List[Any]							=      len(get_test_dummy_examples()		)
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
														_UpperCAmelCase    :						Tuple							=      DummyBeamDataset(cache_dir=A_  ,   beam_runner="DirectRunner"		)
														builder.download_and_prepare()
														self.assertTrue(
														    os.path.exists(
														        os.path.join(A_  ,   builder.name  ,   "default"  ,   "0.0.0"  ,   f'{builder.name}-train.arrow'		)		)		)
														self.assertDictEqual(builder.info.features  ,   datasets.Features({"content": datasets.Value("string"		)}		)		)
														_UpperCAmelCase    :						int							=      builder.as_dataset()
														self.assertEqual(dset["train"].num_rows  ,   A_		)
														self.assertEqual(dset["train"].info.splits["train"].num_examples  ,   A_		)
														self.assertDictEqual(dset["train"][0]  ,   get_test_dummy_examples()[0][1]		)
														self.assertDictEqual(
														    dset["train"][expected_num_examples - 1]  ,   get_test_dummy_examples()[expected_num_examples - 1][1]		)
														self.assertTrue(
														    os.path.exists(os.path.join(A_  ,   builder.name  ,   "default"  ,   "0.0.0"  ,   "dataset_info.json"		)		)		)
														del dset


						@require_beam
						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										import apache_beam as beam

										_UpperCAmelCase    :						Optional[int]							=      beam.io.parquetio.WriteToParquet

										_UpperCAmelCase    :						Optional[int]							=      len(get_test_dummy_examples()		)
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
														_UpperCAmelCase    :						List[str]							=      DummyBeamDataset(cache_dir=A_  ,   beam_runner="DirectRunner"		)
														with patch("apache_beam.io.parquetio.WriteToParquet"		) as write_parquet_mock:
																		_UpperCAmelCase    :						Tuple							=      partial(A_  ,   num_shards=2		)
																		builder.download_and_prepare()
														self.assertTrue(
														    os.path.exists(
														        os.path.join(
														            A_  ,   builder.name  ,   "default"  ,   "0.0.0"  ,   f'{builder.name}-train-00000-of-00002.arrow'		)		)		)
														self.assertTrue(
														    os.path.exists(
														        os.path.join(
														            A_  ,   builder.name  ,   "default"  ,   "0.0.0"  ,   f'{builder.name}-train-00000-of-00002.arrow'		)		)		)
														self.assertDictEqual(builder.info.features  ,   datasets.Features({"content": datasets.Value("string"		)}		)		)
														_UpperCAmelCase    :						Dict							=      builder.as_dataset()
														self.assertEqual(dset["train"].num_rows  ,   A_		)
														self.assertEqual(dset["train"].info.splits["train"].num_examples  ,   A_		)
														# Order is not preserved when sharding, so we just check that all the elements are there
														self.assertListEqual(sorted(dset["train"]["content"]		)  ,   sorted(["foo", "bar", "foobar"]		)		)
														self.assertTrue(
														    os.path.exists(os.path.join(A_  ,   builder.name  ,   "default"  ,   "0.0.0"  ,   "dataset_info.json"		)		)		)
														del dset


						@require_beam
						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										with tempfile.TemporaryDirectory() as tmp_cache_dir:
														_UpperCAmelCase    :						int							=      DummyBeamDataset(cache_dir=A_		)
														self.assertRaises(datasets.builder.MissingBeamOptions  ,   builder.download_and_prepare		)





						@require_beam
						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						Optional[Any]							=      len(get_test_nested_examples()		)
										with tempfile.TemporaryDirectory() as tmp_cache_dir:
														_UpperCAmelCase    :						Optional[int]							=      NestedBeamDataset(cache_dir=A_  ,   beam_runner="DirectRunner"		)
														builder.download_and_prepare()
														self.assertTrue(
														    os.path.exists(
														        os.path.join(A_  ,   builder.name  ,   "default"  ,   "0.0.0"  ,   f'{builder.name}-train.arrow'		)		)		)
														self.assertDictEqual(
														    builder.info.features  ,   datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"		)}		)}		)		)
														_UpperCAmelCase    :						List[str]							=      builder.as_dataset()
														self.assertEqual(dset["train"].num_rows  ,   A_		)
														self.assertEqual(dset["train"].info.splits["train"].num_examples  ,   A_		)
														self.assertDictEqual(dset["train"][0]  ,   get_test_nested_examples()[0][1]		)
														self.assertDictEqual(
														    dset["train"][expected_num_examples - 1]  ,   get_test_nested_examples()[expected_num_examples - 1][1]		)
														self.assertTrue(
														    os.path.exists(os.path.join(A_  ,   builder.name  ,   "default"  ,   "0.0.0"  ,   "dataset_info.json"		)		)		)
														del dset





import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import is_speech_available, is_vision_available
from transformers.testing_utils import require_torch


if is_vision_available():
					from transformers import TvltImageProcessor

if is_speech_available():
					from transformers import TvltFeatureExtractor

from transformers import TvltProcessor

@require_torch
class 			a					(		unittest.TestCase			):


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						int							=      "ZinengTang/tvlt-base"
										_UpperCAmelCase    :						int							=      tempfile.mkdtemp()


						def     _UpperCAmelCase			(    self  ,   **A_		):







										'''simple docstring'''


										return TvltImageProcessor.from_pretrained(self.checkpoint  ,   **A_		)


						def     _UpperCAmelCase			(    self  ,   **A_		):







										'''simple docstring'''


										return TvltFeatureExtractor.from_pretrained(self.checkpoint  ,   **A_		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										shutil.rmtree(self.tmpdirname		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						Dict							=      self.get_image_processor()
										_UpperCAmelCase    :						Optional[int]							=      self.get_feature_extractor()

										_UpperCAmelCase    :						Tuple							=      TvltProcessor(image_processor=A_  ,   feature_extractor=A_		)
										processor.save_pretrained(self.tmpdirname		)
										_UpperCAmelCase    :						str							=      TvltProcessor.from_pretrained(self.tmpdirname		)

										self.assertIsInstance(processor.feature_extractor  ,   A_		)
										self.assertIsInstance(processor.image_processor  ,   A_		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						List[Any]							=      self.get_image_processor()
										_UpperCAmelCase    :						int							=      self.get_feature_extractor()

										_UpperCAmelCase    :						Tuple							=      TvltProcessor(image_processor=A_  ,   feature_extractor=A_		)

										_UpperCAmelCase    :						List[str]							=      np.ones([12000]		)

										_UpperCAmelCase    :						int							=      feature_extractor(A_  ,   return_tensors="np"		)
										_UpperCAmelCase    :						int							=      processor(audio=A_  ,   return_tensors="np"		)

										for key in audio_dict.keys():
														self.assertAlmostEqual(audio_dict[key].sum()  ,   input_processor[key].sum()  ,   delta=1e-2		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						List[str]							=      self.get_image_processor()
										_UpperCAmelCase    :						int							=      self.get_feature_extractor()

										_UpperCAmelCase    :						Optional[Any]							=      TvltProcessor(image_processor=A_  ,   feature_extractor=A_		)

										_UpperCAmelCase    :						Union[str, Any]							=      np.ones([3, 224, 224]		)

										_UpperCAmelCase    :						Tuple							=      image_processor(A_  ,   return_tensors="np"		)
										_UpperCAmelCase    :						List[str]							=      processor(images=A_  ,   return_tensors="np"		)

										for key in image_dict.keys():
														self.assertAlmostEqual(image_dict[key].sum()  ,   input_processor[key].sum()  ,   delta=1e-2		)


						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						List[str]							=      self.get_image_processor()
										_UpperCAmelCase    :						Any							=      self.get_feature_extractor()

										_UpperCAmelCase    :						Dict							=      TvltProcessor(image_processor=A_  ,   feature_extractor=A_		)

										_UpperCAmelCase    :						str							=      np.ones([12000]		)
										_UpperCAmelCase    :						Optional[Any]							=      np.ones([3, 224, 224]		)

										_UpperCAmelCase    :						List[Any]							=      processor(audio=A_  ,   images=A_		)

										self.assertListEqual(list(inputs.keys()		)  ,   ["audio_values", "audio_mask", "pixel_values", "pixel_mask"]		)

										# test if it raises when no input is passed
										with pytest.raises(A_		):
														processor()





						def     _UpperCAmelCase			(    self		):







										'''simple docstring'''


										_UpperCAmelCase    :						Optional[Any]							=      self.get_image_processor()
										_UpperCAmelCase    :						int							=      self.get_feature_extractor()

										_UpperCAmelCase    :						str							=      TvltProcessor(image_processor=A_  ,   feature_extractor=A_		)

										self.assertListEqual(
										    processor.model_input_names  ,   image_processor.model_input_names + feature_extractor.model_input_names  ,   msg="`processor` and `image_processor`+`feature_extractor` model input names do not match"  ,   )


