


'''simple docstring'''






import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


lowercase							='http://www.mocksite.com/file1.txt'
lowercase							='"text": ["foo", "foo"]'
lowercase							='6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'






class  __magic_name__							:
       UpperCAmelCase										=2_0_0
       UpperCAmelCase										={"Content-Length": "100"}
       UpperCAmelCase										={}






       def       lowerCAmelCase		(      self  ,							**snake_case)						->  Optional[Any]:




          '''simple docstring'''
          return [bytes(snake_case  ,							'utf-8')]






def lowerCamelCase__      (						*__lowerCamelCase	:							List[str]	,   **__lowerCamelCase	:							str							):







   '''simple docstring'''




   return MockResponse()





@pytest.mark.parametrize('urls_type'	,   [str, list, dict]							)
def lowerCamelCase__      (						__lowerCamelCase	:							Optional[int]	,   __lowerCamelCase	:							Union[str, Any]	,   __lowerCamelCase	:							int							):







   '''simple docstring'''




   import requests

   monkeypatch.setattr(__lowerCamelCase	,   'request'	,   __lowerCamelCase							)

   _UpperCAmelCase       :       Dict							   =URL
   if issubclass(__lowerCamelCase	,   __lowerCamelCase							):
      _UpperCAmelCase       :       Union[str, Any]							   =url
   elif issubclass(__lowerCamelCase	,   __lowerCamelCase							):
      _UpperCAmelCase       :       Union[str, Any]							   =[url]
   elif issubclass(__lowerCamelCase	,   __lowerCamelCase							):
      _UpperCAmelCase       :       Optional[int]							   ={'train': url}
   _UpperCAmelCase       :       Optional[Any]							   ='dummy'
   _UpperCAmelCase       :       Tuple							   ='downloads'
   _UpperCAmelCase       :       Union[str, Any]							   =tmp_path
   _UpperCAmelCase       :       str							   =DownloadConfig(
       cache_dir=os.path.join(__lowerCamelCase	,   __lowerCamelCase							)	,   use_etag=__lowerCamelCase	,   )
   _UpperCAmelCase       :       Tuple							   =DownloadManager(dataset_name=__lowerCamelCase	,   download_config=__lowerCamelCase							)
   _UpperCAmelCase       :       Tuple							   =dl_manager.download(__lowerCamelCase							)
   _UpperCAmelCase       :       List[str]							   =urls
   for downloaded_paths in [downloaded_paths]:
      if isinstance(__lowerCamelCase	,   __lowerCamelCase							):
         _UpperCAmelCase       :       str							   =[downloaded_paths]
         _UpperCAmelCase       :       Union[str, Any]							   =[urls]
      elif isinstance(__lowerCamelCase	,   __lowerCamelCase							):
         assert "train" in downloaded_paths.keys()
         _UpperCAmelCase       :       Optional[int]							   =downloaded_paths.values()
         _UpperCAmelCase       :       Optional[int]							   =urls.values()
      assert downloaded_paths
      for downloaded_path, input_url in zip(__lowerCamelCase	,   __lowerCamelCase							):
         assert downloaded_path == dl_manager.downloaded_paths[input_url]
         _UpperCAmelCase       :       Optional[Any]							   =Path(__lowerCamelCase							)
         _UpperCAmelCase       :       Optional[int]							   =downloaded_path.parts
         assert parts[-1] == HASH
         assert parts[-2] == cache_subdir
         assert downloaded_path.exists()
         _UpperCAmelCase       :       List[Any]							   =downloaded_path.read_text()
         assert content == CONTENT
         _UpperCAmelCase       :       int							   =downloaded_path.with_suffix('.json'							)
         assert metadata_downloaded_path.exists()
         _UpperCAmelCase       :       List[str]							   =json.loads(metadata_downloaded_path.read_text()							)
         assert metadata_content == {"url": URL, "etag": None}





@pytest.mark.parametrize('paths_type'	,   [str, list, dict]							)
def lowerCamelCase__      (						__lowerCamelCase	:							int	,   __lowerCamelCase	:							List[str]	,   __lowerCamelCase	:							List[str]							):







   '''simple docstring'''




   _UpperCAmelCase       :       List[str]							   =str(__lowerCamelCase							)
   if issubclass(__lowerCamelCase	,   __lowerCamelCase							):
      _UpperCAmelCase       :       int							   =filename
   elif issubclass(__lowerCamelCase	,   __lowerCamelCase							):
      _UpperCAmelCase       :       Any							   =[filename]
   elif issubclass(__lowerCamelCase	,   __lowerCamelCase							):
      _UpperCAmelCase       :       int							   ={'train': filename}
   _UpperCAmelCase       :       int							   ='dummy'
   _UpperCAmelCase       :       str							   =xz_file.parent
   _UpperCAmelCase       :       List[str]							   ='extracted'
   _UpperCAmelCase       :       Union[str, Any]							   =DownloadConfig(
       cache_dir=__lowerCamelCase	,   use_etag=__lowerCamelCase	,   )
   _UpperCAmelCase       :       str							   =DownloadManager(dataset_name=__lowerCamelCase	,   download_config=__lowerCamelCase							)
   _UpperCAmelCase       :       Any							   =dl_manager.extract(__lowerCamelCase							)
   _UpperCAmelCase       :       int							   =paths
   for extracted_paths in [extracted_paths]:
      if isinstance(__lowerCamelCase	,   __lowerCamelCase							):
         _UpperCAmelCase       :       Any							   =[extracted_paths]
         _UpperCAmelCase       :       Union[str, Any]							   =[paths]
      elif isinstance(__lowerCamelCase	,   __lowerCamelCase							):
         assert "train" in extracted_paths.keys()
         _UpperCAmelCase       :       List[str]							   =extracted_paths.values()
         _UpperCAmelCase       :       List[str]							   =paths.values()
      assert extracted_paths
      for extracted_path, input_path in zip(__lowerCamelCase	,   __lowerCamelCase							):
         assert extracted_path == dl_manager.extracted_paths[input_path]
         _UpperCAmelCase       :       List[Any]							   =Path(__lowerCamelCase							)
         _UpperCAmelCase       :       Dict							   =extracted_path.parts
         assert parts[-1] == hash_url_to_filename(__lowerCamelCase	,   etag=__lowerCamelCase							)
         assert parts[-2] == extracted_subdir
         assert extracted_path.exists()
         _UpperCAmelCase       :       Union[str, Any]							   =extracted_path.read_text()
         _UpperCAmelCase       :       Tuple							   =text_file.read_text()
         assert extracted_file_content == expected_file_content





def lowerCamelCase__      (						__lowerCamelCase	:							int	,   __lowerCamelCase	:							Union[str, Any]							):







   '''simple docstring'''




   assert path.endswith('.jsonl'							)
   for num_items, line in enumerate(__lowerCamelCase	,   start=1							):
      _UpperCAmelCase       :       int							   =json.loads(line.decode('utf-8'							)							)
      assert item.keys() == {"col_1", "col_2", "col_3"}
   assert num_items == 4





@pytest.mark.parametrize('archive_jsonl'	,   ['tar_jsonl_path', 'zip_jsonl_path']							)
def lowerCamelCase__      (						__lowerCamelCase	:							Optional[Any]	,   __lowerCamelCase	:							Optional[Any]							):







   '''simple docstring'''




   _UpperCAmelCase       :       Tuple							   =request.getfixturevalue(__lowerCamelCase							)
   _UpperCAmelCase       :       Dict							   =DownloadManager()
   for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(__lowerCamelCase							)	,   start=1							):
      _test_jsonl(__lowerCamelCase	,   __lowerCamelCase							)
   assert num_jsonl == 2





@pytest.mark.parametrize('archive_nested_jsonl'	,   ['tar_nested_jsonl_path', 'zip_nested_jsonl_path']							)
def lowerCamelCase__      (						__lowerCamelCase	:							Optional[Any]	,   __lowerCamelCase	:							str							):







   '''simple docstring'''




   _UpperCAmelCase       :       int							   =request.getfixturevalue(__lowerCamelCase							)
   _UpperCAmelCase       :       Optional[Any]							   =DownloadManager()
   for num_tar, (path, file) in enumerate(dl_manager.iter_archive(__lowerCamelCase							)	,   start=1							):
      for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(__lowerCamelCase							)	,   start=1							):
         _test_jsonl(__lowerCamelCase	,   __lowerCamelCase							)
   assert num_tar == 1
   assert num_jsonl == 2





def lowerCamelCase__      (						__lowerCamelCase	:							Tuple							):







   '''simple docstring'''




   _UpperCAmelCase       :       Tuple							   =DownloadManager()
   for num_file, file in enumerate(dl_manager.iter_files(__lowerCamelCase							)	,   start=1							):
      assert os.path.basename(__lowerCamelCase							) == ("test.txt" if num_file == 1 else "train.txt")
   assert num_file == 2





'''simple docstring'''






import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import is_speech_available, is_vision_available
from transformers.testing_utils import require_torch


if is_vision_available():
 from transformers import TvltImageProcessor

if is_speech_available():
 from transformers import TvltFeatureExtractor

from transformers import TvltProcessor






@require_torch
class  __magic_name__							(		unittest.TestCase    ):






       def       lowerCAmelCase		(      self)						->  str:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   ='ZinengTang/tvlt-base'
          _UpperCAmelCase       :       Dict							   =tempfile.mkdtemp()






       def       lowerCAmelCase		(      self  ,							**snake_case)						->  Union[str, Any]:




          '''simple docstring'''
          return TvltImageProcessor.from_pretrained(self.checkpoint  ,							**snake_case)






       def       lowerCAmelCase		(      self  ,							**snake_case)						->  Dict:




          '''simple docstring'''
          return TvltFeatureExtractor.from_pretrained(self.checkpoint  ,							**snake_case)






       def       lowerCAmelCase		(      self)						->  Any:




          '''simple docstring'''
          shutil.rmtree(self.tmpdirname)






       def       lowerCAmelCase		(      self)						->  Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase       :       Any							   =self.get_image_processor()
          _UpperCAmelCase       :       Optional[Any]							   =self.get_feature_extractor()

          _UpperCAmelCase       :       str							   =TvltProcessor(image_processor=snake_case  ,							feature_extractor=snake_case)
          processor.save_pretrained(self.tmpdirname)
          _UpperCAmelCase       :       str							   =TvltProcessor.from_pretrained(self.tmpdirname)

          self.assertIsInstance(processor.feature_extractor  ,							snake_case)
          self.assertIsInstance(processor.image_processor  ,							snake_case)






       def       lowerCAmelCase		(      self)						->  Union[str, Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       str							   =self.get_image_processor()
          _UpperCAmelCase       :       List[Any]							   =self.get_feature_extractor()

          _UpperCAmelCase       :       str							   =TvltProcessor(image_processor=snake_case  ,							feature_extractor=snake_case)

          _UpperCAmelCase       :       Optional[int]							   =np.ones([1_2_0_0_0])

          _UpperCAmelCase       :       str							   =feature_extractor(snake_case  ,							return_tensors='np')
          _UpperCAmelCase       :       Dict							   =processor(audio=snake_case  ,							return_tensors='np')

          for key in audio_dict.keys():
             self.assertAlmostEqual(audio_dict[key].sum()  ,							input_processor[key].sum()  ,							delta=1E-2)






       def       lowerCAmelCase		(      self)						->  int:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   =self.get_image_processor()
          _UpperCAmelCase       :       int							   =self.get_feature_extractor()

          _UpperCAmelCase       :       int							   =TvltProcessor(image_processor=snake_case  ,							feature_extractor=snake_case)

          _UpperCAmelCase       :       Union[str, Any]							   =np.ones([3, 2_2_4, 2_2_4])

          _UpperCAmelCase       :       Optional[Any]							   =image_processor(snake_case  ,							return_tensors='np')
          _UpperCAmelCase       :       List[Any]							   =processor(images=snake_case  ,							return_tensors='np')

          for key in image_dict.keys():
             self.assertAlmostEqual(image_dict[key].sum()  ,							input_processor[key].sum()  ,							delta=1E-2)






       def       lowerCAmelCase		(      self)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase       :       List[str]							   =self.get_image_processor()
          _UpperCAmelCase       :       Dict							   =self.get_feature_extractor()

          _UpperCAmelCase       :       Optional[int]							   =TvltProcessor(image_processor=snake_case  ,							feature_extractor=snake_case)

          _UpperCAmelCase       :       Optional[int]							   =np.ones([1_2_0_0_0])
          _UpperCAmelCase       :       str							   =np.ones([3, 2_2_4, 2_2_4])

          _UpperCAmelCase       :       Optional[int]							   =processor(audio=snake_case  ,							images=snake_case)

          self.assertListEqual(list(inputs.keys())  ,							['audio_values', 'audio_mask', 'pixel_values', 'pixel_mask'])

          # test if it raises when no input is passed
          with pytest.raises(snake_case):
             processor()






       def       lowerCAmelCase		(      self)						->  Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase       :       Optional[int]							   =self.get_image_processor()
          _UpperCAmelCase       :       Tuple							   =self.get_feature_extractor()

          _UpperCAmelCase       :       Dict							   =TvltProcessor(image_processor=snake_case  ,							feature_extractor=snake_case)

          self.assertListEqual(
              processor.model_input_names  ,							image_processor.model_input_names + feature_extractor.model_input_names  ,							msg='`processor` and `image_processor`+`feature_extractor` model input names do not match'  ,							)

