





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case           =			logging.get_logger(__name__)

_snake_case           =			{
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}





class  lowerCAmelCase							(       _snake_case ):
  __lowerCamelCase		=      'megatron-bert'







  def __init__(      self  :str   ,							_lowercase  :Any=2_90_56   ,							_lowercase  :Union[str, Any]=10_24   ,							_lowercase  :str=24   ,							_lowercase  :Any=16   ,							_lowercase  :Union[str, Any]=40_96   ,							_lowercase  :List[str]="gelu"   ,							_lowercase  :Union[str, Any]=0.1   ,							_lowercase  :List[str]=0.1   ,							_lowercase  :Dict=5_12   ,							_lowercase  :Optional[Any]=2   ,							_lowercase  :List[str]=0.02   ,							_lowercase  :Optional[int]=1e-12   ,							_lowercase  :List[Any]=0   ,							_lowercase  :Optional[int]="absolute"   ,							_lowercase  :int=True   ,							**_lowercase  :List[Any]   ,							):






     '''simple docstring'''





     super().__init__(pad_token_id=UpperCamelCase__   ,							**UpperCamelCase__ )

     lowercase__			       =  vocab_size
     lowercase__			       =  hidden_size
     lowercase__			       =  num_hidden_layers
     lowercase__			       =  num_attention_heads
     lowercase__			       =  hidden_act
     lowercase__			       =  intermediate_size
     lowercase__			       =  hidden_dropout_prob
     lowercase__			       =  attention_probs_dropout_prob
     lowercase__			       =  max_position_embeddings
     lowercase__			       =  type_vocab_size
     lowercase__			       =  initializer_range
     lowercase__			       =  layer_norm_eps
     lowercase__			       =  position_embedding_type
     lowercase__			       =  use_cache













import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch


def _A       (   __magic_name__	):  # picklable for multiprocessing
   return x.sum()


def _A       (   __magic_name__	):  # picklable for multiprocessing
   return i + 1





@dataclass
class  lowerCAmelCase							:
  __lowerCamelCase		=      42
  __lowerCamelCase		=      42





class  lowerCAmelCase							(       lowercase_ ):







  def   UpperCAmelCase					(      self  :Optional[Any] ):






     '''simple docstring'''





     lowercase__			       =  {}
     lowercase__			       =  []
     lowercase__			       =  1
     lowercase__			       =  [1, 2]
     lowercase__			       =  {"a": 1, "b": 2}
     lowercase__			       =  {"a": [1, 2], "b": [3, 4]}
     lowercase__			       =  {"a": {"1": 1}, "b": 2}
     lowercase__			       =  {"a": 1, "b": 2, "c": 3, "d": 4}
     lowercase__			       =  {}
     lowercase__			       =  []
     lowercase__			       =  2
     lowercase__			       =  [2, 3]
     lowercase__			       =  {"a": 2, "b": 3}
     lowercase__			       =  {"a": [2, 3], "b": [4, 5]}
     lowercase__			       =  {"a": {"1": 2}, "b": 3}
     lowercase__			       =  {"a": 2, "b": 3, "c": 4, "d": 5}
     self.assertEqual(map_nested(_lowercase   ,							_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase )   ,							_lowercase )

     lowercase__			       =  2
     self.assertEqual(map_nested(_lowercase   ,							_lowercase   ,							num_proc=_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase   ,							num_proc=_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase   ,							num_proc=_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase   ,							num_proc=_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase   ,							num_proc=_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase   ,							num_proc=_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase   ,							num_proc=_lowercase )   ,							_lowercase )
     self.assertEqual(map_nested(_lowercase   ,							_lowercase   ,							num_proc=_lowercase )   ,							_lowercase )

     lowercase__			       =  {"a": np.eye(2 ), "b": np.zeros(3 ), "c": np.ones(2 )}
     lowercase__			       =  {"a": 2, "b": 0, "c": 2}
     lowercase__			       =  {
         "a": np.eye(2 ).astype(_lowercase ),
         "b": np.zeros(3 ).astype(_lowercase ),
         "c": np.ones(2 ).astype(_lowercase ),
     }
     self.assertEqual(map_nested(_lowercase   ,							_lowercase   ,							map_numpy=_lowercase )   ,							_lowercase )
     self.assertEqual(
         {k: v.tolist() for k, v in map_nested(_lowercase   ,							_lowercase   ,							map_numpy=_lowercase ).items()}   ,							{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}   ,							)
     self.assertEqual(map_nested(_lowercase   ,							_lowercase   ,							map_numpy=_lowercase   ,							num_proc=_lowercase )   ,							_lowercase )
     self.assertEqual(
         {k: v.tolist() for k, v in map_nested(_lowercase   ,							_lowercase   ,							map_numpy=_lowercase   ,							num_proc=_lowercase ).items()}   ,							{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}   ,							)
     with self.assertRaises(_lowercase ):  # can't pickle a local lambda
        map_nested(lambda _lowercase : x + 1   ,							_lowercase   ,							num_proc=_lowercase )







  def   UpperCAmelCase					(      self  :List[Any] ):






     '''simple docstring'''





     lowercase__			       =  {"a": 1, "b": 2}
     lowercase__			       =  {"a": 3, "b": 4}
     lowercase__			       =  {"a": 5, "b": 6}
     lowercase__			       =  sorted([("a", (1, 3, 5)), ("b", (2, 4, 6))] )
     self.assertEqual(sorted(zip_dict(_lowercase   ,							_lowercase   ,							_lowercase ) )   ,							_lowercase )







  def   UpperCAmelCase					(      self  :Union[str, Any] ):






     '''simple docstring'''





     class  lowerCAmelCase							:
       __lowerCamelCase		=      'bar'
     lowercase__			       =  Foo()
     self.assertEqual(foo.my_attr   ,							"bar" )
     with temporary_assignment(_lowercase   ,							"my_attr"   ,							"BAR" ):
        self.assertEqual(foo.my_attr   ,							"BAR" )
     self.assertEqual(foo.my_attr   ,							"bar" )


@pytest.mark.parametrize(
    "iterable_length, num_proc, expected_num_proc"  , [
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]  , )
def _A       (   __magic_name__  , __magic_name__  , __magic_name__	):
   with patch("datasets.utils.py_utils._single_map_nested"	) as mock_single_map_nested, patch(
       "datasets.parallel.parallel.Pool"	) as mock_multiprocessing_pool:
      lowercase__			       =  {f'''{i}''': i for i in range(__magic_name__	)}
      lowercase__			       =  map_nested(lambda __magic_name__	: x + 10  , __magic_name__  , num_proc=__magic_name__  , parallel_min_length=16	)
      if expected_num_proc == 1:
         assert mock_single_map_nested.called
         assert not mock_multiprocessing_pool.called
      else:
         assert not mock_single_map_nested.called
         assert mock_multiprocessing_pool.called
         assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc





class  lowerCAmelCase							(       lowercase_ ):







  @require_tf
  def   UpperCAmelCase					(      self  :List[Any] ):






     '''simple docstring'''





     import tensorflow as tf
     from tensorflow.keras import layers

     lowercase__			       =  layers.Dense(2 )

     def gen_random_output():
        lowercase__			       =  tf.random.uniform((1, 3) )
        return model(_lowercase ).numpy()

     with temp_seed(42   ,							set_tensorflow=_lowercase ):
        lowercase__			       =  gen_random_output()
     with temp_seed(42   ,							set_tensorflow=_lowercase ):
        lowercase__			       =  gen_random_output()
     lowercase__			       =  gen_random_output()

     np.testing.assert_equal(_lowercase   ,							_lowercase )
     self.assertGreater(np.abs(outa - outa ).sum()   ,							0 )







  @require_torch
  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     import torch

     def gen_random_output():
        lowercase__			       =  torch.nn.Linear(3   ,							2 )
        lowercase__			       =  torch.rand(1   ,							3 )
        return model(_lowercase ).detach().numpy()

     with temp_seed(42   ,							set_pytorch=_lowercase ):
        lowercase__			       =  gen_random_output()
     with temp_seed(42   ,							set_pytorch=_lowercase ):
        lowercase__			       =  gen_random_output()
     lowercase__			       =  gen_random_output()

     np.testing.assert_equal(_lowercase   ,							_lowercase )
     self.assertGreater(np.abs(outa - outa ).sum()   ,							0 )







  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     def gen_random_output():
        return np.random.rand(1   ,							3 )

     with temp_seed(42 ):
        lowercase__			       =  gen_random_output()
     with temp_seed(42 ):
        lowercase__			       =  gen_random_output()
     lowercase__			       =  gen_random_output()

     np.testing.assert_equal(_lowercase   ,							_lowercase )
     self.assertGreater(np.abs(outa - outa ).sum()   ,							0 )


@pytest.mark.parametrize("input_data"  , [{}]	)
def _A       (   __magic_name__	):
   lowercase__			       =  NestedDataStructure(__magic_name__	).data
   assert output_data == input_data


@pytest.mark.parametrize(
    "data, expected_output"  , [
        ({}, []),
        ([], []),
        ("foo", ["foo"]),
        (["foo", "bar"], ["foo", "bar"]),
        ([["foo", "bar"]], ["foo", "bar"]),
        ([[["foo"], ["bar"]]], ["foo", "bar"]),
        ([[["foo"], "bar"]], ["foo", "bar"]),
        ({"a": 1, "b": 2}, [1, 2]),
        ({"a": [1, 2], "b": [3, 4]}, [1, 2, 3, 4]),
        ({"a": [[1, 2]], "b": [[3, 4]]}, [1, 2, 3, 4]),
        ({"a": [[1, 2]], "b": [3, 4]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [[[3], [4]]]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [[3, 4]]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [3, 4]}, [1, 2, 3, 4]),
        ({"a": [[[1], [2]]], "b": [3, [4]]}, [1, 2, 3, 4]),
        ({"a": {"1": 1}, "b": 2}, [1, 2]),
        ({"a": {"1": [1]}, "b": 2}, [1, 2]),
        ({"a": {"1": [1]}, "b": [2]}, [1, 2]),
    ]  , )
def _A       (   __magic_name__  , __magic_name__	):
   lowercase__			       =  NestedDataStructure(__magic_name__	).flatten()
   assert output == expected_output


def _A       (   ):
   lowercase__			       =  A(x=1  , y="foobar"	)
   lowercase__			       =  {"x": 1, "y": "foobar"}
   assert asdict(__magic_name__	) == expected_output

   lowercase__			       =  {"a": {"b": A(x=10  , y="foo"	)}, "c": [A(x=20  , y="bar"	)]}
   lowercase__			       =  {"a": {"b": {"x": 10, "y": "foo"}}, "c": [{"x": 20, "y": "bar"}]}
   assert asdict(__magic_name__	) == expected_output

   with pytest.raises(__magic_name__	):
      asdict([1, A(x=10  , y="foo"	)]	)


def _A       (   __magic_name__	):
   return text.split()


def _A       (   __magic_name__	):
   yield (time.time(), content)
   time.sleep(2	)
   yield (time.time(), content)







def _A       (   ):
   with Pool(2	) as pool:
      lowercase__			       =  list(iflatmap_unordered(__magic_name__  , _split_text  , kwargs_iterable=[{"text": "hello there"}] * 10	)	)
      assert out.count("hello"	) == 10
      assert out.count("there"	) == 10
      assert len(__magic_name__	) == 20

   # check multiprocess from pathos (uses dill for pickling)
   with multiprocess.Pool(2	) as pool:
      lowercase__			       =  list(iflatmap_unordered(__magic_name__  , _split_text  , kwargs_iterable=[{"text": "hello there"}] * 10	)	)
      assert out.count("hello"	) == 10
      assert out.count("there"	) == 10
      assert len(__magic_name__	) == 20

   # check that we get items as fast as possible
   with Pool(2	) as pool:
      lowercase__			       =  []
      for yield_time, content in iflatmap_unordered(
          __magic_name__  , _aseconds_generator_of_aitems_with_timing  , kwargs_iterable=[{"content": "a"}, {"content": "b"}]	):
         assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
         out.append(__magic_name__	)
      assert out.count("a"	) == 2
      assert out.count("b"	) == 2
      assert len(__magic_name__	) == 4






