





import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
				from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm



def A_     (		A__    )  ->		Optional[int]:
							if is_torch_version('<'		,				'2.0.0'    ) or not hasattr(A__		,				'_dynamo'    ):
														return False
							return isinstance(A__		,				torch._dynamo.eval_frame.OptimizedModule    )



def A_     (		A__		,				A__ = True    )  ->		int:
							a__						:     Optional[Any]			=    (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

							a__						:     Union[str, Any]			=    is_compiled_module(A__    )
							if is_compiled:
														a__						:     List[str]			=    model
														a__						:     Dict			=    model._orig_mod

							if is_deepspeed_available():
														options += (DeepSpeedEngine,)

							while isinstance(A__		,				A__    ):
														a__						:     str			=    model.module

							if not keep_fpaa_wrapper:
														a__						:     Union[str, Any]			=    getattr(A__		,				'forward'    )
														a__						:     List[Any]			=    model.__dict__.pop('_original_forward'		,				A__    )
														if original_forward is not None:
																					while hasattr(A__		,				'__wrapped__'    ):
																												a__						:     int			=    forward.__wrapped__
																												if forward == original_forward:
																																			break
																					a__						:     List[Any]			=    forward
														if getattr(A__		,				'_converted_to_transformer_engine'		,				A__    ):
																					convert_model(A__		,				to_transformer_engine=A__    )

							if is_compiled:
														a__						:     List[str]			=    model
														a__						:     Any			=    compiled_model

							return model



def A_     (		)  ->		int:
							PartialState().wait_for_everyone()



def A_     (		A__		,				A__    )  ->		Dict:
							if PartialState().distributed_type == DistributedType.TPU:
														xm.save(A__		,				A__    )
							elif PartialState().local_process_index == 0:
														torch.save(A__		,				A__    )



@contextmanager
def A_     (		**A__    )  ->		Any:
							for key, value in kwargs.items():
														a__						:     Optional[int]			=    str(A__    )

							yield

							for key in kwargs:
														if key.upper() in os.environ:
																					del os.environ[key.upper()]



def A_     (		A__    )  ->		List[str]:
							if not hasattr(A__		,				'__qualname__'    ) and not hasattr(A__		,				'__name__'    ):
														a__						:     Dict			=    getattr(A__		,				'__class__'		,				A__    )
							if hasattr(A__		,				'__qualname__'    ):
														return obj.__qualname__
							if hasattr(A__		,				'__name__'    ):
														return obj.__name__
							return str(A__    )



def A_     (		A__		,				A__    )  ->		Dict:
							for key, value in source.items():
														if isinstance(A__		,				A__    ):
																					a__						:     Optional[Any]			=    destination.setdefault(A__		,				{}    )
																					merge_dicts(A__		,				A__    )
														else:
																					a__						:     Optional[int]			=    value

							return destination



def A_     (		A__ = None    )  ->		bool:
							if port is None:
														a__						:     List[Any]			=    2_9500
							with socket.socket(socket.AF_INET		,				socket.SOCK_STREAM    ) as s:
														return s.connect_ex(('localhost', port)    ) == 0









import unittest

import numpy as np

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
    from transformers.models.bert.modeling_flax_bert import (
        FlaxBertForMaskedLM,
        FlaxBertForMultipleChoice,
        FlaxBertForNextSentencePrediction,
        FlaxBertForPreTraining,
        FlaxBertForQuestionAnswering,
        FlaxBertForSequenceClassification,
        FlaxBertForTokenClassification,
        FlaxBertModel,
    )





class 			A__   (							unittest.TestCase     ):




   """simple docstring"""



   def __init__( self ,  lowercase ,  lowercase=13 ,  lowercase=7 ,  lowercase=True ,  lowercase=True ,  lowercase=True ,  lowercase=True ,  lowercase=99 ,  lowercase=32 ,  lowercase=5 ,  lowercase=4 ,  lowercase=37 ,  lowercase="gelu" ,  lowercase=0.1 ,  lowercase=0.1 ,  lowercase=512 ,  lowercase=16 ,  lowercase=2 ,  lowercase=0.02 ,  lowercase=4 ,  )      ->						Tuple:







          '''simple docstring'''

          a__						:     str			=    parent
          a__						:     Optional[Any]			=    batch_size
          a__						:     str			=    seq_length
          a__						:     int			=    is_training
          a__						:     str			=    use_attention_mask
          a__						:     List[str]			=    use_token_type_ids
          a__						:     Optional[Any]			=    use_labels
          a__						:     List[Any]			=    vocab_size
          a__						:     Tuple			=    hidden_size
          a__						:     Dict			=    num_hidden_layers
          a__						:     List[str]			=    num_attention_heads
          a__						:     int			=    intermediate_size
          a__						:     Any			=    hidden_act
          a__						:     Optional[int]			=    hidden_dropout_prob
          a__						:     Tuple			=    attention_probs_dropout_prob
          a__						:     Tuple			=    max_position_embeddings
          a__						:     Optional[int]			=    type_vocab_size
          a__						:     List[Any]			=    type_sequence_label_size
          a__						:     Union[str, Any]			=    initializer_range
          a__						:     str			=    num_choices

   def 							__lowercase		( self)      ->						Optional[Any]:







          '''simple docstring'''

          a__						:     str			=    ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size)

          a__						:     Dict			=    None
          if self.use_attention_mask:
                 a__						:     Optional[int]			=    random_attention_mask([self.batch_size, self.seq_length])

          a__						:     Dict			=    None
          if self.use_token_type_ids:
                 a__						:     int			=    ids_tensor([self.batch_size, self.seq_length] ,  self.type_vocab_size)

          a__						:     Dict			=    BertConfig(
              vocab_size=self.vocab_size ,  hidden_size=self.hidden_size ,  num_hidden_layers=self.num_hidden_layers ,  num_attention_heads=self.num_attention_heads ,  intermediate_size=self.intermediate_size ,  hidden_act=self.hidden_act ,  hidden_dropout_prob=self.hidden_dropout_prob ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob ,  max_position_embeddings=self.max_position_embeddings ,  type_vocab_size=self.type_vocab_size ,  is_decoder=lowercase ,  initializer_range=self.initializer_range ,  )

          return config, input_ids, token_type_ids, attention_mask

   def 							__lowercase		( self)      ->						Optional[Any]:







          '''simple docstring'''

          a__						:     Tuple			=    self.prepare_config_and_inputs()
          a__			,							a__			,							a__			,							a__						:     Optional[int]			=    config_and_inputs
          a__						:     str			=    {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
          return config, inputs_dict

   def 							__lowercase		( self)      ->						Optional[int]:







          '''simple docstring'''

          a__						:     Union[str, Any]			=    self.prepare_config_and_inputs()
          a__			,							a__			,							a__			,							a__						:     int			=    config_and_inputs

          a__						:     str			=    True
          a__						:     Any			=    floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
          a__						:     List[str]			=    ids_tensor([self.batch_size, self.seq_length] ,  vocab_size=2)

          return (
              config,
              input_ids,
              attention_mask,
              encoder_hidden_states,
              encoder_attention_mask,
          )





@require_flax
class 			A__   (							__UpperCAmelCase			,  unittest.TestCase     ):




   """simple docstring"""



   __A       :    Optional[Any]	  =  True

   __A       :    Tuple	  =  (
       (
           FlaxBertModel,
           FlaxBertForPreTraining,
           FlaxBertForMaskedLM,
           FlaxBertForMultipleChoice,
           FlaxBertForQuestionAnswering,
           FlaxBertForNextSentencePrediction,
           FlaxBertForSequenceClassification,
           FlaxBertForTokenClassification,
           FlaxBertForQuestionAnswering,
       )
       if is_flax_available()
       else ()
   )

   def 							__lowercase		( self)      ->						Tuple:







          '''simple docstring'''

          a__						:     Optional[Any]			=    FlaxBertModelTester(self)

   @slow
   def 							__lowercase		( self)      ->						Optional[Any]:







          '''simple docstring'''

          a__						:     Union[str, Any]			=    FlaxBertModel.from_pretrained('bert-base-cased')
          a__						:     Optional[Any]			=    model(np.ones((1, 1)))
          self.assertIsNotNone(lowercase)


