




import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch







def lowerCamelCase__       (      _lowercase				):  # picklable for multiprocessing

     '''simple docstring'''


     return x.sum()







def lowerCamelCase__       (      _lowercase				):  # picklable for multiprocessing

     '''simple docstring'''


     return i + 1

@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				42
    lowerCAmelCase        =				42


class   __a(					_a       ):






    """simple docstring"""





    def 							a__				(				self  )			->		List[str]:
         UpperCAmelCase_      :							Tuple       =      {}
         UpperCAmelCase_      :							Optional[int]       =      []
         UpperCAmelCase_      :							Optional[Any]       =      1
         UpperCAmelCase_      :							Optional[Any]       =      [1, 2]
         UpperCAmelCase_      :							Dict       =      {'''a''': 1, '''b''': 2}
         UpperCAmelCase_      :							List[str]       =      {'''a''': [1, 2], '''b''': [3, 4]}
         UpperCAmelCase_      :							List[Any]       =      {'''a''': {'''1''': 1}, '''b''': 2}
         UpperCAmelCase_      :							Dict       =      {'''a''': 1, '''b''': 2, '''c''': 3, '''d''': 4}
         UpperCAmelCase_      :							Optional[Any]       =      {}
         UpperCAmelCase_      :							Optional[int]       =      []
         UpperCAmelCase_      :							Union[str, Any]       =      2
         UpperCAmelCase_      :							Union[str, Any]       =      [2, 3]
         UpperCAmelCase_      :							str       =      {'''a''': 2, '''b''': 3}
         UpperCAmelCase_      :							Optional[Any]       =      {'''a''': [2, 3], '''b''': [4, 5]}
         UpperCAmelCase_      :							Tuple       =      {'''a''': {'''1''': 2}, '''b''': 3}
         UpperCAmelCase_      :							List[str]       =      {'''a''': 2, '''b''': 3, '''c''': 4, '''d''': 5}
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )

         UpperCAmelCase_      :							Any       =      2
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )

         UpperCAmelCase_      :							Tuple       =      {'''a''': np.eye(2  ), '''b''': np.zeros(3  ), '''c''': np.ones(2  )}
         UpperCAmelCase_      :							List[str]       =      {'''a''': 2, '''b''': 0, '''c''': 2}
         UpperCAmelCase_      :							Any       =      {
             '''a''': np.eye(2  ).astype(_SCREAMING_SNAKE_CASE  ),
             '''b''': np.zeros(3  ).astype(_SCREAMING_SNAKE_CASE  ),
             '''c''': np.ones(2  ).astype(_SCREAMING_SNAKE_CASE  ),
         }
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,map_numpy=_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(
             {k: v.tolist() for k, v in map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,map_numpy=_SCREAMING_SNAKE_CASE  ).items()}       ,{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}       ,)
         self.assertEqual(map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,map_numpy=_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )
         self.assertEqual(
             {k: v.tolist() for k, v in map_nested(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,map_numpy=_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  ).items()}       ,{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}       ,)
         with self.assertRaises(_SCREAMING_SNAKE_CASE  ):  # can't pickle a local lambda
              map_nested(lambda _SCREAMING_SNAKE_CASE  : x + 1       ,_SCREAMING_SNAKE_CASE       ,num_proc=_SCREAMING_SNAKE_CASE  )





    def 							a__				(				self  )			->		Dict:
         UpperCAmelCase_      :							int       =      {'''a''': 1, '''b''': 2}
         UpperCAmelCase_      :							str       =      {'''a''': 3, '''b''': 4}
         UpperCAmelCase_      :							Tuple       =      {'''a''': 5, '''b''': 6}
         UpperCAmelCase_      :							Union[str, Any]       =      sorted([('''a''', (1, 3, 5)), ('''b''', (2, 4, 6))]  )
         self.assertEqual(sorted(zip_dict(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )  )       ,_SCREAMING_SNAKE_CASE  )





    def 							a__				(				self  )			->		Tuple:

         class   __a:






             """simple docstring"""





             lowerCAmelCase        =				'''bar'''


         UpperCAmelCase_      :							Union[str, Any]       =      Foo()
         self.assertEqual(foo.my_attr       ,'''bar'''  )
         with temporary_assignment(_SCREAMING_SNAKE_CASE       ,'''my_attr'''       ,'''BAR'''  ):
              self.assertEqual(foo.my_attr       ,'''BAR'''  )
         self.assertEqual(foo.my_attr       ,'''bar'''  )







@pytest.mark.parametrize(
    '''iterable_length, num_proc, expected_num_proc'''			,						[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]			,						)
def lowerCamelCase__       (      _lowercase			,						_lowercase			,						_lowercase				):

     '''simple docstring'''


     with patch('''datasets.utils.py_utils._single_map_nested'''				) as mock_single_map_nested, patch(
         '''datasets.parallel.parallel.Pool'''				) as mock_multiprocessing_pool:
          UpperCAmelCase_      :							List[str]       =      {f'''{i}''': i for i in range(_lowercase				)}
          UpperCAmelCase_      :							Optional[int]       =      map_nested(lambda _lowercase				: x + 10			,						_lowercase			,						num_proc=_lowercase			,						parallel_min_length=16				)
          if expected_num_proc == 1:
               assert mock_single_map_nested.called
               assert not mock_multiprocessing_pool.called
          else:
               assert not mock_single_map_nested.called
               assert mock_multiprocessing_pool.called
               assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc

class   __a(					_a       ):






    """simple docstring"""





    @require_tf
    def 							a__				(				self  )			->		str:
         import tensorflow as tf
         from tensorflow.keras import layers

         UpperCAmelCase_      :							Tuple       =      layers.Dense(2  )

         def gen_random_output():
              UpperCAmelCase_      :							Union[str, Any]       =      tf.random.uniform((1, 3)  )
              return model(_SCREAMING_SNAKE_CASE  ).numpy()

         with temp_seed(42       ,set_tensorflow=_SCREAMING_SNAKE_CASE  ):
              UpperCAmelCase_      :							Optional[Any]       =      gen_random_output()
         with temp_seed(42       ,set_tensorflow=_SCREAMING_SNAKE_CASE  ):
              UpperCAmelCase_      :							List[Any]       =      gen_random_output()
         UpperCAmelCase_      :							str       =      gen_random_output()

         np.testing.assert_equal(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )
         self.assertGreater(np.abs(outa - outa  ).sum()       ,0  )





    @require_torch
    def 							a__				(				self  )			->		Optional[int]:
         import torch

         def gen_random_output():
              UpperCAmelCase_      :							Optional[int]       =      torch.nn.Linear(3       ,2  )
              UpperCAmelCase_      :							int       =      torch.rand(1       ,3  )
              return model(_SCREAMING_SNAKE_CASE  ).detach().numpy()

         with temp_seed(42       ,set_pytorch=_SCREAMING_SNAKE_CASE  ):
              UpperCAmelCase_      :							Any       =      gen_random_output()
         with temp_seed(42       ,set_pytorch=_SCREAMING_SNAKE_CASE  ):
              UpperCAmelCase_      :							Tuple       =      gen_random_output()
         UpperCAmelCase_      :							str       =      gen_random_output()

         np.testing.assert_equal(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )
         self.assertGreater(np.abs(outa - outa  ).sum()       ,0  )





    def 							a__				(				self  )			->		Tuple:
         def gen_random_output():
              return np.random.rand(1       ,3  )

         with temp_seed(42  ):
              UpperCAmelCase_      :							Tuple       =      gen_random_output()
         with temp_seed(42  ):
              UpperCAmelCase_      :							Optional[int]       =      gen_random_output()
         UpperCAmelCase_      :							Union[str, Any]       =      gen_random_output()

         np.testing.assert_equal(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )
         self.assertGreater(np.abs(outa - outa  ).sum()       ,0  )







@pytest.mark.parametrize('''input_data'''			,						[{}]				)
def lowerCamelCase__       (      _lowercase				):

     '''simple docstring'''


     UpperCAmelCase_      :							Optional[Any]       =      NestedDataStructure(_lowercase				).data
     assert output_data == input_data







@pytest.mark.parametrize(
    '''data, expected_output'''			,						[
        ({}, []),
        ([], []),
        ('''foo''', ['''foo''']),
        (['''foo''', '''bar'''], ['''foo''', '''bar''']),
        ([['''foo''', '''bar''']], ['''foo''', '''bar''']),
        ([[['''foo'''], ['''bar''']]], ['''foo''', '''bar''']),
        ([[['''foo'''], '''bar''']], ['''foo''', '''bar''']),
        ({'''a''': 1, '''b''': 2}, [1, 2]),
        ({'''a''': [1, 2], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[1, 2]], '''b''': [[3, 4]]}, [1, 2, 3, 4]),
        ({'''a''': [[1, 2]], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [[3, 4]]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [3, [4]]}, [1, 2, 3, 4]),
        ({'''a''': {'''1''': 1}, '''b''': 2}, [1, 2]),
        ({'''a''': {'''1''': [1]}, '''b''': 2}, [1, 2]),
        ({'''a''': {'''1''': [1]}, '''b''': [2]}, [1, 2]),
    ]			,						)
def lowerCamelCase__       (      _lowercase			,						_lowercase				):

     '''simple docstring'''


     UpperCAmelCase_      :							Tuple       =      NestedDataStructure(_lowercase				).flatten()
     assert output == expected_output







def lowerCamelCase__       (      ):

     '''simple docstring'''


     UpperCAmelCase_      :							Any       =      A(x=1			,						y='''foobar'''				)
     UpperCAmelCase_      :							int       =      {'''x''': 1, '''y''': '''foobar'''}
     assert asdict(_lowercase				) == expected_output

     UpperCAmelCase_      :							Any       =      {'''a''': {'''b''': A(x=10			,						y='''foo'''				)}, '''c''': [A(x=20			,						y='''bar'''				)]}
     UpperCAmelCase_      :							Any       =      {'''a''': {'''b''': {'''x''': 10, '''y''': '''foo'''}}, '''c''': [{'''x''': 20, '''y''': '''bar'''}]}
     assert asdict(_lowercase				) == expected_output

     with pytest.raises(_lowercase				):
          asdict([1, A(x=10			,						y='''foo'''				)]				)







def lowerCamelCase__       (      _lowercase				):

     '''simple docstring'''


     return text.split()







def lowerCamelCase__       (      _lowercase				):

     '''simple docstring'''


     yield (time.time(), content)
     time.sleep(2				)
     yield (time.time(), content)







def lowerCamelCase__       (      ):

     '''simple docstring'''


     with Pool(2				) as pool:
          UpperCAmelCase_      :							Dict       =      list(iflatmap_unordered(_lowercase			,						_split_text			,						kwargs_iterable=[{'''text''': '''hello there'''}] * 10				)				)
          assert out.count('''hello'''				) == 10
          assert out.count('''there'''				) == 10
          assert len(_lowercase				) == 20

     # check multiprocess from pathos (uses dill for pickling)
     with multiprocess.Pool(2				) as pool:
          UpperCAmelCase_      :							str       =      list(iflatmap_unordered(_lowercase			,						_split_text			,						kwargs_iterable=[{'''text''': '''hello there'''}] * 10				)				)
          assert out.count('''hello'''				) == 10
          assert out.count('''there'''				) == 10
          assert len(_lowercase				) == 20

     # check that we get items as fast as possible
     with Pool(2				) as pool:
          UpperCAmelCase_      :							int       =      []
          for yield_time, content in iflatmap_unordered(
              _lowercase			,						_aseconds_generator_of_aitems_with_timing			,						kwargs_iterable=[{'''content''': '''a'''}, {'''content''': '''b'''}]				):
               assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
               out.append(_lowercase				)
          assert out.count('''a'''				) == 2
          assert out.count('''b'''				) == 2
          assert len(_lowercase				) == 4





import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


__a          =			logging.get_logger(__name__)







def lowerCamelCase__       (      _lowercase				):

     '''simple docstring'''


     UpperCAmelCase_      :							str       =      r'''\w+[.]\d+'''
     UpperCAmelCase_      :							int       =      re.findall(_lowercase			,						_lowercase				)
     for pat in pats:
          UpperCAmelCase_      :							List[Any]       =      key.replace(_lowercase			,						'''_'''.join(pat.split('''.'''				)				)				)
     return key







def lowerCamelCase__       (      _lowercase			,						_lowercase			,						_lowercase				):

     '''simple docstring'''


     UpperCAmelCase_      :							Dict       =      pt_tuple_key[:-1] + ('''scale''',)
     if (
         any('''norm''' in str_ for str_ in pt_tuple_key				)
         and (pt_tuple_key[-1] == "bias")
         and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
         and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
     ):
          UpperCAmelCase_      :							List[Any]       =      pt_tuple_key[:-1] + ('''scale''',)
          return renamed_pt_tuple_key, pt_tensor
     elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
          UpperCAmelCase_      :							Any       =      pt_tuple_key[:-1] + ('''scale''',)
          return renamed_pt_tuple_key, pt_tensor

     # embedding
     if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
          UpperCAmelCase_      :							Tuple       =      pt_tuple_key[:-1] + ('''embedding''',)
          return renamed_pt_tuple_key, pt_tensor

     # conv layer
     UpperCAmelCase_      :							List[Any]       =      pt_tuple_key[:-1] + ('''kernel''',)
     if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
          UpperCAmelCase_      :							List[Any]       =      pt_tensor.transpose(2			,						3			,						1			,						0				)
          return renamed_pt_tuple_key, pt_tensor

     # linear layer
     UpperCAmelCase_      :							Dict       =      pt_tuple_key[:-1] + ('''kernel''',)
     if pt_tuple_key[-1] == "weight":
          UpperCAmelCase_      :							Tuple       =      pt_tensor.T
          return renamed_pt_tuple_key, pt_tensor

     # old PyTorch layer norm weight
     UpperCAmelCase_      :							Any       =      pt_tuple_key[:-1] + ('''weight''',)
     if pt_tuple_key[-1] == "gamma":
          return renamed_pt_tuple_key, pt_tensor

     # old PyTorch layer norm bias
     UpperCAmelCase_      :							Tuple       =      pt_tuple_key[:-1] + ('''bias''',)
     if pt_tuple_key[-1] == "beta":
          return renamed_pt_tuple_key, pt_tensor

     return pt_tuple_key, pt_tensor







def lowerCamelCase__       (      _lowercase			,						_lowercase			,						_lowercase=42				):

     '''simple docstring'''


     UpperCAmelCase_      :							Union[str, Any]       =      {k: v.numpy() for k, v in pt_state_dict.items()}

     # Step 2: Since the model is stateless, get random Flax params
     UpperCAmelCase_      :							str       =      flax_model.init_weights(PRNGKey(_lowercase				)				)

     UpperCAmelCase_      :							List[Any]       =      flatten_dict(_lowercase				)
     UpperCAmelCase_      :							int       =      {}

     # Need to change some parameters name to match Flax names
     for pt_key, pt_tensor in pt_state_dict.items():
          UpperCAmelCase_      :							Optional[int]       =      rename_key(_lowercase				)
          UpperCAmelCase_      :							List[str]       =      tuple(renamed_pt_key.split('''.'''				)				)

          # Correctly rename weight parameters
          UpperCAmelCase_,   UpperCAmelCase_      :							Any       =      rename_key_and_reshape_tensor(_lowercase			,						_lowercase			,						_lowercase				)

          if flax_key in random_flax_state_dict:
               if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                    raise ValueError(
                        f'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
                        f'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''				)

        # also add unexpected weight so that warning is thrown
          UpperCAmelCase_      :							int       =      jnp.asarray(_lowercase				)

     return unflatten_dict(_lowercase				)