





'''simple docstring'''






import os
import sys
import unittest


A__    :       Optional[int]              =os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
A__    :       List[str]              =os.path.join(git_repo_path, '''src''', '''transformers''')

A__    :       Any              ='''
{0} = None
'''

A__    :       int              ='''
class {0}(metaclass=DummyObject):
    _backends = {1}

    def __init__(self, *args, **kwargs):
        requires_backends(self, {1})
'''


A__    :       Optional[int]              ='''
def {0}(*args, **kwargs):
    requires_backends({0}, {1})
'''




class 			UpperCAmelCase					(      unittest.TestCase       ):




      def     lowercase__   (      self      :			Optional[int]  )    ->				int:
            _lowerCAmelCase												=							find_backend("""    _import_structure[\"models.albert\"].append(\"AlbertTokenizerFast\")"""  )
            self.assertIsNone(__snake_case  )

            _lowerCAmelCase												=							find_backend("""    if not is_tokenizers_available():"""  )
            self.assertEqual(__snake_case      ,      """tokenizers"""  )

            _lowerCAmelCase												=							find_backend("""    if not is_tensorflow_text_available():"""  )
            self.assertEqual(__snake_case      ,      """tensorflow_text"""  )

            _lowerCAmelCase												=							find_backend("""    if not (is_sentencepiece_available() and is_tokenizers_available()):"""  )
            self.assertEqual(__snake_case      ,      """sentencepiece_and_tokenizers"""  )

            _lowerCAmelCase												=							find_backend(
                """    if not (is_sentencepiece_available() and is_tensorflow_text_available()):"""  )
            self.assertEqual(__snake_case      ,      """sentencepiece_and_tensorflow_text"""  )

            _lowerCAmelCase												=							find_backend(
                """    if not (is_sentencepiece_available() and is_tokenizers_available() and is_vision_available()):"""  )
            self.assertEqual(__snake_case      ,      """sentencepiece_and_tokenizers_and_vision"""  )




      def     lowercase__   (      self      :			Dict  )    ->				Dict:
            _lowerCAmelCase												=							read_init()
            # We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
            self.assertIn("""torch"""      ,      __snake_case  )
            self.assertIn("""tensorflow_text"""      ,      __snake_case  )
            self.assertIn("""sentencepiece_and_tokenizers"""      ,      __snake_case  )

            # Likewise, we can't assert on the exact content of a key
            self.assertIn("""BertModel"""      ,      objects["""torch"""]  )
            self.assertIn("""TFBertModel"""      ,      objects["""tf"""]  )
            self.assertIn("""FlaxBertModel"""      ,      objects["""flax"""]  )
            self.assertIn("""BertModel"""      ,      objects["""torch"""]  )
            self.assertIn("""TFBertTokenizer"""      ,      objects["""tensorflow_text"""]  )
            self.assertIn("""convert_slow_tokenizer"""      ,      objects["""sentencepiece_and_tokenizers"""]  )




      def     lowercase__   (      self      :			int  )    ->				Any:
            _lowerCAmelCase												=							create_dummy_object("""CONSTANT"""      ,      """'torch'"""  )
            self.assertEqual(__snake_case      ,      """\nCONSTANT = None\n"""  )

            _lowerCAmelCase												=							create_dummy_object("""function"""      ,      """'torch'"""  )
            self.assertEqual(
                __snake_case      ,      """\ndef function(*args, **kwargs):\n    requires_backends(function, 'torch')\n"""  )

            _lowerCAmelCase												=							"""
class FakeClass(metaclass=DummyObject):
    _backends = 'torch'

    def __init__(self, *args, **kwargs):
        requires_backends(self, 'torch')
"""
            _lowerCAmelCase												=							create_dummy_object("""FakeClass"""      ,      """'torch'"""  )
            self.assertEqual(__snake_case      ,      __snake_case  )




      def     lowercase__   (      self      :			Union[str, Any]  )    ->				Union[str, Any]:
            _lowerCAmelCase												=							"""# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..utils import DummyObject, requires_backends


CONSTANT = None


def function(*args, **kwargs):
    requires_backends(function, [\"torch\"])


class FakeClass(metaclass=DummyObject):
    _backends = [\"torch\"]

    def __init__(self, *args, **kwargs):
        requires_backends(self, [\"torch\"])
"""
            _lowerCAmelCase												=							create_dummy_files({"""torch""": ["""CONSTANT""", """function""", """FakeClass"""]}  )
            self.assertEqual(dummy_files["""torch"""]      ,      __snake_case  )













'''simple docstring'''






import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


A__    :       str              =5_00_00
A__    :       Optional[int]              =50_00

A__       ,						A__    :       Optional[int]              =os.path.split(__file__)
A__    :       Tuple              =os.path.join(RESULTS_BASEPATH, '''results''', RESULTS_FILENAME.replace('''.py''', '''.json'''))
@get_duration
def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase  ):





      """simple docstring"""


      for i in range(lowerCAmelCase  ):
            _lowerCAmelCase												=							dataset[i]
@get_duration
def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase  ):





      """simple docstring"""


      for i in range(0							,     len(lowerCAmelCase  )							,     lowerCAmelCase  ):
            _lowerCAmelCase												=							dataset[i : i + batch_size]
@get_duration
def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase  ):





      """simple docstring"""


      with dataset.formatted_as(type=lowerCAmelCase  ):
            for i in range(lowerCAmelCase  ):
                  _lowerCAmelCase												=							dataset[i]
@get_duration
def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase  ):





      """simple docstring"""


      with dataset.formatted_as(type=lowerCAmelCase  ):
            for i in range(0							,     lowerCAmelCase							,     lowerCAmelCase  ):
                  _lowerCAmelCase												=							dataset[i : i + batch_size]
def        UpperCamelCase__				(		):





      """simple docstring"""


      _lowerCAmelCase												=							{"""num examples""": SPEED_TEST_N_EXAMPLES}
      _lowerCAmelCase												=							[
          (read, {"""length""": SMALL_TEST}),
          (read, {"""length""": SPEED_TEST_N_EXAMPLES}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 10}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 1_00}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 10_00}),
          (read_formatted, {"""type""": """numpy""", """length""": SMALL_TEST}),
          (read_formatted, {"""type""": """pandas""", """length""": SMALL_TEST}),
          (read_formatted, {"""type""": """torch""", """length""": SMALL_TEST}),
          (read_formatted, {"""type""": """tensorflow""", """length""": SMALL_TEST}),
          (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 10}),
          (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 10_00}),
      ]

      _lowerCAmelCase												=							[
          (read, {"""length""": SMALL_TEST}),
          (read, {"""length""": SPEED_TEST_N_EXAMPLES}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 10}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 1_00}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 10_00}),
          (read_formatted, {"""type""": """numpy""", """length""": SMALL_TEST}),
          (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 10}),
          (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 10_00}),
      ]
      with tempfile.TemporaryDirectory() as tmp_dir:
            print("""generating dataset"""  )
            _lowerCAmelCase												=							datasets.Features(
                {"""list""": datasets.Sequence(datasets.Value("""float32"""  )  ), """numbers""": datasets.Value("""float32"""  )}  )
            _lowerCAmelCase												=							generate_example_dataset(
                os.path.join(lowerCAmelCase							,     """dataset.arrow"""  )							,     lowerCAmelCase							,     num_examples=lowerCAmelCase							,     seq_shapes={"""list""": (1_00,)}							,     )
            print("""first set of iterations"""  )
            for func, kwargs in functions:
                  print(func.__name__							,     str(lowerCAmelCase  )  )
                  _lowerCAmelCase												=							func(lowerCAmelCase							,     **lowerCAmelCase  )

            print("""shuffling dataset"""  )
            _lowerCAmelCase												=							dataset.shuffle()
            print("""Second set of iterations (after shuffling"""  )
            for func, kwargs in functions_shuffled:
                  print("""shuffled """							,     func.__name__							,     str(lowerCAmelCase  )  )
                  _lowerCAmelCase												=							func(
                      lowerCAmelCase							,     **lowerCAmelCase  )

      with open(lowerCAmelCase							,     """wb"""  ) as f:
            f.write(json.dumps(lowerCAmelCase  ).encode("""utf-8"""  )  )


if __name__ == "__main__":  # useful to run the profiler
      benchmark_iterating()






