



"""simple docstring"""






import argparse

import torch

from transformers import (
    SpeechTaConfig,
    SpeechTaFeatureExtractor,
    SpeechTaForSpeechToSpeech,
    SpeechTaForSpeechToText,
    SpeechTaForTextToSpeech,
    SpeechTaProcessor,
    SpeechTaTokenizer,
    logging,
)
from transformers.tokenization_utils import AddedToken


logging.set_verbosity_info()
lowerCAmelCase__      =    logging.get_logger('''transformers.models.speecht5''')

lowerCAmelCase__      =    {
    '''speech_encoder_prenet.layer_norm''': '''speecht5.encoder.prenet.feature_projection.layer_norm''',
    '''speech_encoder_prenet.post_extract_proj''': '''speecht5.encoder.prenet.feature_projection.projection''',
    '''speech_encoder_prenet.pos_conv.0''': '''speecht5.encoder.prenet.pos_conv_embed.conv''',
    '''speech_encoder_prenet.mask_emb''': '''speecht5.encoder.prenet.masked_spec_embed''',
}
lowerCAmelCase__      =    {
    '''text_encoder_prenet.encoder_prenet.0''': '''speecht5.encoder.prenet.embed_tokens''',
    '''text_encoder_prenet.encoder_prenet.1.alpha''': '''speecht5.encoder.prenet.encode_positions.alpha''',
}
lowerCAmelCase__      =    {
    '''speech_decoder_prenet.decoder_prenet.0.0.prenet.0.0''': '''speecht5.decoder.prenet.layers.0''',
    '''speech_decoder_prenet.decoder_prenet.0.0.prenet.1.0''': '''speecht5.decoder.prenet.layers.1''',
    '''speech_decoder_prenet.decoder_prenet.0.1''': '''speecht5.decoder.prenet.final_layer''',
    '''speech_decoder_prenet.decoder_prenet.1.alpha''': '''speecht5.decoder.prenet.encode_positions.alpha''',
    '''speech_decoder_prenet.spkembs_layer.0''': '''speecht5.decoder.prenet.speaker_embeds_layer''',
}
lowerCAmelCase__      =    {
    '''speech_decoder_postnet.feat_out''': '''speech_decoder_postnet.feat_out''',
    '''speech_decoder_postnet.prob_out''': '''speech_decoder_postnet.prob_out''',
    '''speech_decoder_postnet.postnet.postnet.0.0''': '''speech_decoder_postnet.layers.0.conv''',
    '''speech_decoder_postnet.postnet.postnet.0.1''': '''speech_decoder_postnet.layers.0.batch_norm''',
    '''speech_decoder_postnet.postnet.postnet.1.0''': '''speech_decoder_postnet.layers.1.conv''',
    '''speech_decoder_postnet.postnet.postnet.1.1''': '''speech_decoder_postnet.layers.1.batch_norm''',
    '''speech_decoder_postnet.postnet.postnet.2.0''': '''speech_decoder_postnet.layers.2.conv''',
    '''speech_decoder_postnet.postnet.postnet.2.1''': '''speech_decoder_postnet.layers.2.batch_norm''',
    '''speech_decoder_postnet.postnet.postnet.3.0''': '''speech_decoder_postnet.layers.3.conv''',
    '''speech_decoder_postnet.postnet.postnet.3.1''': '''speech_decoder_postnet.layers.3.batch_norm''',
    '''speech_decoder_postnet.postnet.postnet.4.0''': '''speech_decoder_postnet.layers.4.conv''',
    '''speech_decoder_postnet.postnet.postnet.4.1''': '''speech_decoder_postnet.layers.4.batch_norm''',
}
lowerCAmelCase__      =    {
    '''text_decoder_prenet.embed_tokens''': '''speecht5.decoder.prenet.embed_tokens''',
}
lowerCAmelCase__      =    {
    '''text_decoder_postnet.output_projection''': '''text_decoder_postnet.lm_head''',
}
lowerCAmelCase__      =    {
    '''encoder.layers.*.self_attn.k_proj''': '''speecht5.encoder.wrapped_encoder.layers.*.attention.k_proj''',
    '''encoder.layers.*.self_attn.v_proj''': '''speecht5.encoder.wrapped_encoder.layers.*.attention.v_proj''',
    '''encoder.layers.*.self_attn.q_proj''': '''speecht5.encoder.wrapped_encoder.layers.*.attention.q_proj''',
    '''encoder.layers.*.self_attn.out_proj''': '''speecht5.encoder.wrapped_encoder.layers.*.attention.out_proj''',
    '''encoder.layers.*.self_attn_layer_norm''': '''speecht5.encoder.wrapped_encoder.layers.*.layer_norm''',
    '''encoder.layers.*.fc1''': '''speecht5.encoder.wrapped_encoder.layers.*.feed_forward.intermediate_dense''',
    '''encoder.layers.*.fc2''': '''speecht5.encoder.wrapped_encoder.layers.*.feed_forward.output_dense''',
    '''encoder.layers.*.final_layer_norm''': '''speecht5.encoder.wrapped_encoder.layers.*.final_layer_norm''',
    '''encoder.layer_norm''': '''speecht5.encoder.wrapped_encoder.layer_norm''',
    '''encoder.pos_emb.pe_k''': '''speecht5.encoder.wrapped_encoder.embed_positions.pe_k''',
}
lowerCAmelCase__      =    {
    '''decoder.layers.*.self_attn.k_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.self_attn.k_proj''',
    '''decoder.layers.*.self_attn.v_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.self_attn.v_proj''',
    '''decoder.layers.*.self_attn.q_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.self_attn.q_proj''',
    '''decoder.layers.*.self_attn.out_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.self_attn.out_proj''',
    '''decoder.layers.*.self_attn_layer_norm''': '''speecht5.decoder.wrapped_decoder.layers.*.self_attn_layer_norm''',
    '''decoder.layers.*.encoder_attn.k_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.k_proj''',
    '''decoder.layers.*.encoder_attn.v_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.v_proj''',
    '''decoder.layers.*.encoder_attn.q_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.q_proj''',
    '''decoder.layers.*.encoder_attn.out_proj''': '''speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.out_proj''',
    '''decoder.layers.*.encoder_attn_layer_norm''': '''speecht5.decoder.wrapped_decoder.layers.*.encoder_attn_layer_norm''',
    '''decoder.layers.*.fc1''': '''speecht5.decoder.wrapped_decoder.layers.*.feed_forward.intermediate_dense''',
    '''decoder.layers.*.fc2''': '''speecht5.decoder.wrapped_decoder.layers.*.feed_forward.output_dense''',
    '''decoder.layers.*.final_layer_norm''': '''speecht5.decoder.wrapped_decoder.layers.*.final_layer_norm''',
}
lowerCAmelCase__      =    {
    **MAPPING_SPEECH_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_TEXT_DECODER_PRENET,
    **MAPPING_TEXT_DECODER_POSTNET,
}
lowerCAmelCase__      =    {
    **MAPPING_TEXT_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_SPEECH_DECODER_PRENET,
    **MAPPING_SPEECH_DECODER_POSTNET,
}
lowerCAmelCase__      =    {
    **MAPPING_SPEECH_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_SPEECH_DECODER_PRENET,
    **MAPPING_SPEECH_DECODER_POSTNET,
}
lowerCAmelCase__      =    []
lowerCAmelCase__      =    [
    '''encoder.version''',
    '''encoder.layers.*.norm_k.weight''',
    '''encoder.layers.*.norm_k.bias''',
    '''decoder.version''',
    '''decoder.layers.*.norm_k.weight''',
    '''decoder.layers.*.norm_k.bias''',
    '''decoder.pos_emb.pe_k''',
    '''speech_encoder_prenet.embed_positions._float_tensor''',
    '''text_decoder_prenet.embed_positions._float_tensor''',
]
lowerCAmelCase__      =    IGNORE_KEYS + [
    '''encoder.proj''',
    '''text_encoder_prenet.*''',
    '''speech_decoder_prenet.*''',
    '''speech_decoder_postnet.*''',
]
lowerCAmelCase__      =    IGNORE_KEYS + [
    '''encoder.proj''',
    '''speech_encoder_prenet.*''',
    '''text_decoder_prenet.*''',
    '''text_decoder_postnet.*''',
]
lowerCAmelCase__      =    IGNORE_KEYS + [
    '''encoder.proj''',
    '''text_encoder_prenet.*''',
    '''text_decoder_prenet.*''',
    '''text_decoder_postnet.*''',
]






def 	snake_case_  (				A_ :		Optional[Any],    A_ :		Tuple,    A_ :		List[Any],    A_ :		Union[str, Any],    A_ :		Optional[int]						):


       '''simple docstring'''





       for attribute in key.split('''.'''						):
              _lowerCamelCase						:    int         =			getattr(A_,    A_						)

       if weight_type is not None:
              _lowerCamelCase						:    Optional[int]         =			getattr(A_,    A_						).shape
       else:
              _lowerCamelCase						:    Tuple         =			hf_pointer.shape

       if hf_shape != value.shape:
              raise ValueError(
                  F'''Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'''
                  F''' {value.shape} for {full_name}'''						)

       if weight_type == "weight":
              _lowerCamelCase						:    List[Any]         =			value
       elif weight_type == "weight_g":
              _lowerCamelCase						:    List[str]         =			value
       elif weight_type == "weight_v":
              _lowerCamelCase						:    Tuple         =			value
       elif weight_type == "bias":
              _lowerCamelCase						:    Dict         =			value
       elif weight_type == "running_mean":
              _lowerCamelCase						:    List[Any]         =			value
       elif weight_type == "running_var":
              _lowerCamelCase						:    Dict         =			value
       elif weight_type == "num_batches_tracked":
              _lowerCamelCase						:    Optional[int]         =			value
       else:
              _lowerCamelCase						:    List[str]         =			value

       logger.info(F'''{key + ("." + weight_type if weight_type is not None else "")} was initialized from {full_name}.'''						)






def 	snake_case_  (				A_ :		Dict,    A_ :		Union[str, Any]						):


       '''simple docstring'''





       for key in ignore_keys:
              if key.endswith('''.*'''						):
                     if name.startswith(key[:-1]						):
                            return True
              elif ".*." in key:
                     _lowerCamelCase						:    str         =			key.split('''.*.'''						)
                     if prefix in name and suffix in name:
                            return True
              elif key in name:
                     return True
       return False






def 	snake_case_  (				A_ :		int,    A_ :		List[str],    A_ :		Dict						):


       '''simple docstring'''





       _lowerCamelCase						:    Union[str, Any]         =			[]

       if task == "s2t":
              _lowerCamelCase						:    int         =			hf_model.speechta.encoder.prenet.feature_encoder
              _lowerCamelCase						:    str         =			MAPPING_S2T
              _lowerCamelCase						:    Optional[int]         =			IGNORE_KEYS_S2T
       elif task == "t2s":
              _lowerCamelCase						:    Any         =			None
              _lowerCamelCase						:    Any         =			MAPPING_T2S
              _lowerCamelCase						:    Dict         =			IGNORE_KEYS_T2S
       elif task == "s2s":
              _lowerCamelCase						:    List[Any]         =			hf_model.speechta.encoder.prenet.feature_encoder
              _lowerCamelCase						:    Any         =			MAPPING_S2S
              _lowerCamelCase						:    str         =			IGNORE_KEYS_S2S
       else:
              raise ValueError(F'''Unsupported task: {task}'''						)

       for name, value in fairseq_dict.items():
              if should_ignore(A_,    A_						):
                     logger.info(F'''{name} was ignored'''						)
                     continue

              _lowerCamelCase						:    List[Any]         =			False
              if "conv_layers" in name:
                     load_conv_layer(
                         A_,    A_,    A_,    A_,    hf_model.config.feat_extract_norm == '''group''',    )
                     _lowerCamelCase						:    Any         =			True
              else:
                     for key, mapped_key in MAPPING.items():
                            # mapped_key = "speecht5." + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key

                            if "*" in key:
                                   _lowerCamelCase						:    Tuple         =			key.split('''.*.'''						)
                                   if prefix in name and suffix in name:
                                          _lowerCamelCase						:    Optional[int]         =			suffix

                # if key in name or key.split("w2v_model.")[-1] == name.split(".")[0]:
                            if key in name:
                                   _lowerCamelCase						:    str         =			True
                                   if "*" in mapped_key:
                                          _lowerCamelCase						:    Tuple         =			name.split(A_						)[0].split('''.'''						)[-2]
                                          _lowerCamelCase						:    List[Any]         =			mapped_key.replace('''*''',    A_						)
                                   if "weight_g" in name:
                                          _lowerCamelCase						:    int         =			'''weight_g'''
                                   elif "weight_v" in name:
                                          _lowerCamelCase						:    Any         =			'''weight_v'''
                                   elif "bias" in name:
                                          _lowerCamelCase						:    Dict         =			'''bias'''
                                   elif "weight" in name:
                                          _lowerCamelCase						:    Optional[Any]         =			'''weight'''
                                   elif "running_mean" in name:
                                          _lowerCamelCase						:    Any         =			'''running_mean'''
                                   elif "running_var" in name:
                                          _lowerCamelCase						:    List[str]         =			'''running_var'''
                                   elif "num_batches_tracked" in name:
                                          _lowerCamelCase						:    str         =			'''num_batches_tracked'''
                                   else:
                                          _lowerCamelCase						:    str         =			None
                                   set_recursively(A_,    A_,    A_,    A_,    A_						)
                            continue
              if not is_used:
                     unused_weights.append(A_						)

       logger.warning(F'''Unused weights: {unused_weights}'''						)






def 	snake_case_  (				A_ :		Union[str, Any],    A_ :		Union[str, Any],    A_ :		List[Any],    A_ :		Tuple,    A_ :		List[str]						):


       '''simple docstring'''





       _lowerCamelCase						:    str         =			full_name.split('''conv_layers.'''						)[-1]
       _lowerCamelCase						:    str         =			name.split('''.'''						)
       _lowerCamelCase						:    List[Any]         =			int(items[0]						)
       _lowerCamelCase						:    Tuple         =			int(items[1]						)

       if type_id == 0:
              if "bias" in name:
                     if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
                            raise ValueError(
                                F'''{full_name} has size {value.shape}, but'''
                                F''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''						)
                     _lowerCamelCase						:    Dict         =			value
                     logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''						)
              elif "weight" in name:
                     if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
                            raise ValueError(
                                F'''{full_name} has size {value.shape}, but'''
                                F''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''						)
                     _lowerCamelCase						:    List[Any]         =			value
                     logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''						)
       elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
              if "bias" in name:
                     if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
                            raise ValueError(
                                F'''{full_name} has size {value.shape}, but'''
                                F''' {feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape} was found.'''						)
                     _lowerCamelCase						:    Any         =			value
                     logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''						)
              elif "weight" in name:
                     if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
                            raise ValueError(
                                F'''{full_name} has size {value.shape}, but'''
                                F''' {feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape} was found.'''						)
                     _lowerCamelCase						:    Union[str, Any]         =			value
                     logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''						)
       else:
              unused_weights.append(A_						)






@torch.no_grad()
def 	snake_case_  (				A_ :		Optional[Any],    A_ :		List[str],    A_ :		str,    A_ :		Union[str, Any]=None,    A_ :		Dict=None,    A_ :		Optional[Any]=None,    ):


       '''simple docstring'''





       if config_path is not None:
              _lowerCamelCase						:    Optional[int]         =			SpeechTaConfig.from_pretrained(A_						)
       else:
              _lowerCamelCase						:    int         =			SpeechTaConfig()

       if task == "s2t":
              _lowerCamelCase						:    Tuple         =			config.max_text_positions
              _lowerCamelCase						:    str         =			SpeechTaForSpeechToText(A_						)
       elif task == "t2s":
              _lowerCamelCase						:    Tuple         =			18_76
              _lowerCamelCase						:    Any         =			6_00
              _lowerCamelCase						:    Any         =			config.max_speech_positions
              _lowerCamelCase						:    List[Any]         =			SpeechTaForTextToSpeech(A_						)
       elif task == "s2s":
              _lowerCamelCase						:    Optional[int]         =			18_76
              _lowerCamelCase						:    Optional[int]         =			config.max_speech_positions
              _lowerCamelCase						:    Any         =			SpeechTaForSpeechToSpeech(A_						)
       else:
              raise ValueError(F'''Unknown task name: {task}'''						)

       if vocab_path:
              _lowerCamelCase						:    Optional[Any]         =			SpeechTaTokenizer(A_,    model_max_length=config.max_text_positions						)

              # Mask token behaves like a normal word, i.e. include the space before it
              _lowerCamelCase						:    Any         =			AddedToken('''<mask>''',    lstrip=A_,    rstrip=A_						)
              _lowerCamelCase						:    List[str]         =			mask_token
              tokenizer.add_special_tokens({'''mask_token''': mask_token}						)
              tokenizer.add_tokens(['''<ctc_blank>''']						)

       _lowerCamelCase						:    Union[str, Any]         =			SpeechTaFeatureExtractor()
       _lowerCamelCase						:    Union[str, Any]         =			SpeechTaProcessor(tokenizer=A_,    feature_extractor=A_						)
       processor.save_pretrained(A_						)

       _lowerCamelCase						:    List[str]         =			torch.load(A_						)
       recursively_load_weights(fairseq_checkpoint['''model'''],    A_,    A_						)

       model.save_pretrained(A_						)

       if repo_id:
              print('''Pushing to the hub...'''						)
              processor.push_to_hub(A_						)
              model.push_to_hub(A_						)


if __name__ == "__main__":
   lowerCAmelCase__      =    argparse.ArgumentParser()
   parser.add_argument(
       '''--task''',
       default='''s2t''',
       type=str,
       help='''Type of the SpeechT5 model you\'d like to convert. Should be one of \'s2t\', \'t2s\', \'s2s\'.''',
   )
   parser.add_argument('''--checkpoint_path''', required=True, default=None, type=str, help='''Path to fairseq checkpoint''')
   parser.add_argument('''--vocab_path''', default=None, type=str, help='''Path to SentencePiece model''')
   parser.add_argument('''--config_path''', default=None, type=str, help='''Path to hf config.json of model to convert''')
   parser.add_argument(
       '''--pytorch_dump_folder_path''', required=True, default=None, type=str, help='''Path to the output PyTorch model.'''
   )
   parser.add_argument(
       '''--push_to_hub''', default=None, type=str, help='''Where to upload the converted model on the 🤗 hub.'''
   )

   lowerCAmelCase__      =    parser.parse_args()
   convert_speechta_checkpoint(
       args.task,
       args.checkpoint_path,
       args.pytorch_dump_folder_path,
       args.config_path,
       args.vocab_path,
       args.push_to_hub,
   )







"""simple docstring"""






import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       FlaubertForMultipleChoice,
       FlaubertForQuestionAnswering,
       FlaubertForQuestionAnsweringSimple,
       FlaubertForSequenceClassification,
       FlaubertForTokenClassification,
       FlaubertModel,
       FlaubertWithLMHeadModel,
   )
   from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST






class 	__snake_case	(     _lowercase):




  def __init__(			self					:						List[Any]						, __lowerCAmelCase					:						Dict						, __lowerCAmelCase					:						List[Any]=1_3						, __lowerCAmelCase					:						Any=7						, __lowerCAmelCase					:						Any=True						, __lowerCAmelCase					:						Any=True						, __lowerCAmelCase					:						Union[str, Any]=True						, __lowerCAmelCase					:						List[Any]=True						, __lowerCAmelCase					:						Any=True						, __lowerCAmelCase					:						Union[str, Any]=False						, __lowerCAmelCase					:						List[Any]=False						, __lowerCAmelCase					:						int=False						, __lowerCAmelCase					:						Tuple=2						, __lowerCAmelCase					:						Dict=9_9						, __lowerCAmelCase					:						str=0						, __lowerCAmelCase					:						Optional[Any]=3_2						, __lowerCAmelCase					:						Tuple=5						, __lowerCAmelCase					:						Tuple=4						, __lowerCAmelCase					:						List[str]=0.1						, __lowerCAmelCase					:						Union[str, Any]=0.1						, __lowerCAmelCase					:						Optional[int]=5_1_2						, __lowerCAmelCase					:						Any=1_2						, __lowerCAmelCase					:						Optional[int]=2						, __lowerCAmelCase					:						Any=0.02						, __lowerCAmelCase					:						str=3						, __lowerCAmelCase					:						Optional[int]=4						, __lowerCAmelCase					:						Optional[int]="last"						, __lowerCAmelCase					:						str=None						, __lowerCAmelCase					:						int=None						, ):






         """simple docstring"""






         _lowerCamelCase						:    Dict         =			parent
         _lowerCamelCase						:    List[str]         =			batch_size
         _lowerCamelCase						:    Dict         =			seq_length
         _lowerCamelCase						:    List[Any]         =			is_training
         _lowerCamelCase						:    Dict         =			use_input_lengths
         _lowerCamelCase						:    Tuple         =			use_token_type_ids
         _lowerCamelCase						:    Any         =			use_labels
         _lowerCamelCase						:    Optional[Any]         =			gelu_activation
         _lowerCamelCase						:    Optional[Any]         =			sinusoidal_embeddings
         _lowerCamelCase						:    Dict         =			causal
         _lowerCamelCase						:    Dict         =			asm
         _lowerCamelCase						:    str         =			n_langs
         _lowerCamelCase						:    str         =			vocab_size
         _lowerCamelCase						:    Optional[int]         =			n_special
         _lowerCamelCase						:    Dict         =			hidden_size
         _lowerCamelCase						:    int         =			num_hidden_layers
         _lowerCamelCase						:    str         =			num_attention_heads
         _lowerCamelCase						:    Dict         =			hidden_dropout_prob
         _lowerCamelCase						:    int         =			attention_probs_dropout_prob
         _lowerCamelCase						:    Any         =			max_position_embeddings
         _lowerCamelCase						:    Any         =			type_vocab_size
         _lowerCamelCase						:    Optional[int]         =			type_sequence_label_size
         _lowerCamelCase						:    List[str]         =			initializer_range
         _lowerCamelCase						:    List[Any]         =			num_labels
         _lowerCamelCase						:    Dict         =			num_choices
         _lowerCamelCase						:    str         =			summary_type
         _lowerCamelCase						:    List[str]         =			use_proj
         _lowerCamelCase						:    int         =			scope




  def 						SCREAMING_SNAKE_CASE    (			self					:						Tuple		):






         """simple docstring"""






         _lowerCamelCase						:    List[Any]         =			ids_tensor([self.batch_size, self.seq_length]						, self.vocab_size		)
         _lowerCamelCase						:    Optional[Any]         =			random_attention_mask([self.batch_size, self.seq_length]		)

         _lowerCamelCase						:    Optional[int]         =			None
         if self.use_input_lengths:
                _lowerCamelCase						:    int         =			(
                    ids_tensor([self.batch_size]						, vocab_size=2		) + self.seq_length - 2
                )  # small variation of seq_length

         _lowerCamelCase						:    Union[str, Any]         =			None
         if self.use_token_type_ids:
                _lowerCamelCase						:    str         =			ids_tensor([self.batch_size, self.seq_length]						, self.n_langs		)

         _lowerCamelCase						:    Union[str, Any]         =			None
         _lowerCamelCase						:    List[str]         =			None
         _lowerCamelCase						:    Optional[Any]         =			None
         if self.use_labels:
                _lowerCamelCase						:    Dict         =			ids_tensor([self.batch_size]						, self.type_sequence_label_size		)
                _lowerCamelCase						:    List[str]         =			ids_tensor([self.batch_size, self.seq_length]						, self.num_labels		)
                _lowerCamelCase						:    str         =			ids_tensor([self.batch_size]						, 2		).float()
                _lowerCamelCase						:    Optional[int]         =			ids_tensor([self.batch_size]						, self.num_choices		)

         _lowerCamelCase						:    Tuple         =			self.get_config()

         return (
             config,
             input_ids,
             token_type_ids,
             input_lengths,
             sequence_labels,
             token_labels,
             is_impossible_labels,
             choice_labels,
             input_mask,
         )




  def 						SCREAMING_SNAKE_CASE    (			self					:						Optional[int]		):






         """simple docstring"""






         return FlaubertConfig(
             vocab_size=self.vocab_size						, n_special=self.n_special						, emb_dim=self.hidden_size						, n_layers=self.num_hidden_layers						, n_heads=self.num_attention_heads						, dropout=self.hidden_dropout_prob						, attention_dropout=self.attention_probs_dropout_prob						, gelu_activation=self.gelu_activation						, sinusoidal_embeddings=self.sinusoidal_embeddings						, asm=self.asm						, causal=self.causal						, n_langs=self.n_langs						, max_position_embeddings=self.max_position_embeddings						, initializer_range=self.initializer_range						, summary_type=self.summary_type						, use_proj=self.use_proj						, )




  def 						SCREAMING_SNAKE_CASE    (			self					:						str						, __lowerCAmelCase					:						List[str]						, __lowerCAmelCase					:						Optional[Any]						, __lowerCAmelCase					:						str						, __lowerCAmelCase					:						Optional[Any]						, __lowerCAmelCase					:						Tuple						, __lowerCAmelCase					:						Any						, __lowerCAmelCase					:						Optional[Any]						, __lowerCAmelCase					:						List[str]						, __lowerCAmelCase					:						Tuple						, ):






         """simple docstring"""






         _lowerCamelCase						:    Optional[Any]         =			FlaubertModel(config=__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()
         _lowerCamelCase						:    Union[str, Any]         =			model(__lowerCAmelCase						, lengths=__lowerCAmelCase						, langs=__lowerCAmelCase		)
         _lowerCamelCase						:    str         =			model(__lowerCAmelCase						, langs=__lowerCAmelCase		)
         _lowerCamelCase						:    List[str]         =			model(__lowerCAmelCase		)
         self.parent.assertEqual(result.last_hidden_state.shape						, (self.batch_size, self.seq_length, self.hidden_size)		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Dict						, __lowerCAmelCase					:						str						, __lowerCAmelCase					:						Dict						, __lowerCAmelCase					:						Dict						, __lowerCAmelCase					:						List[str]						, __lowerCAmelCase					:						Optional[Any]						, __lowerCAmelCase					:						Tuple						, __lowerCAmelCase					:						int						, __lowerCAmelCase					:						Optional[int]						, __lowerCAmelCase					:						str						, ):






         """simple docstring"""






         _lowerCamelCase						:    Tuple         =			FlaubertWithLMHeadModel(__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()

         _lowerCamelCase						:    str         =			model(__lowerCAmelCase						, token_type_ids=__lowerCAmelCase						, labels=__lowerCAmelCase		)
         self.parent.assertEqual(result.loss.shape						, ()		)
         self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.seq_length, self.vocab_size)		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						int						, __lowerCAmelCase					:						int						, __lowerCAmelCase					:						int						, __lowerCAmelCase					:						Tuple						, __lowerCAmelCase					:						Any						, __lowerCAmelCase					:						Any						, __lowerCAmelCase					:						str						, __lowerCAmelCase					:						Optional[int]						, __lowerCAmelCase					:						Dict						, __lowerCAmelCase					:						Union[str, Any]						, ):






         """simple docstring"""






         _lowerCamelCase						:    Union[str, Any]         =			FlaubertForQuestionAnsweringSimple(__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()

         _lowerCamelCase						:    List[str]         =			model(__lowerCAmelCase		)

         _lowerCamelCase						:    Union[str, Any]         =			model(__lowerCAmelCase						, start_positions=__lowerCAmelCase						, end_positions=__lowerCAmelCase		)
         self.parent.assertEqual(result.start_logits.shape						, (self.batch_size, self.seq_length)		)
         self.parent.assertEqual(result.end_logits.shape						, (self.batch_size, self.seq_length)		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Union[str, Any]						, __lowerCAmelCase					:						List[Any]						, __lowerCAmelCase					:						Any						, __lowerCAmelCase					:						Optional[Any]						, __lowerCAmelCase					:						List[Any]						, __lowerCAmelCase					:						int						, __lowerCAmelCase					:						int						, __lowerCAmelCase					:						List[Any]						, __lowerCAmelCase					:						Union[str, Any]						, __lowerCAmelCase					:						int						, ):






         """simple docstring"""






         _lowerCamelCase						:    str         =			FlaubertForQuestionAnswering(__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()

         _lowerCamelCase						:    Optional[Any]         =			model(__lowerCAmelCase		)

         _lowerCamelCase						:    Optional[Any]         =			model(
             __lowerCAmelCase						, start_positions=__lowerCAmelCase						, end_positions=__lowerCAmelCase						, cls_index=__lowerCAmelCase						, is_impossible=__lowerCAmelCase						, p_mask=__lowerCAmelCase						, )

         _lowerCamelCase						:    List[str]         =			model(
             __lowerCAmelCase						, start_positions=__lowerCAmelCase						, end_positions=__lowerCAmelCase						, cls_index=__lowerCAmelCase						, is_impossible=__lowerCAmelCase						, )

         ((_lowerCamelCase)      ,   )						:    str         =			result_with_labels.to_tuple()

         _lowerCamelCase						:    Optional[Any]         =			model(__lowerCAmelCase						, start_positions=__lowerCAmelCase						, end_positions=__lowerCAmelCase		)

         ((_lowerCamelCase)      ,   )						:    Union[str, Any]         =			result_with_labels.to_tuple()

         self.parent.assertEqual(result_with_labels.loss.shape						, ()		)
         self.parent.assertEqual(result.start_top_log_probs.shape						, (self.batch_size, model.config.start_n_top)		)
         self.parent.assertEqual(result.start_top_index.shape						, (self.batch_size, model.config.start_n_top)		)
         self.parent.assertEqual(
             result.end_top_log_probs.shape						, (self.batch_size, model.config.start_n_top * model.config.end_n_top)		)
         self.parent.assertEqual(
             result.end_top_index.shape						, (self.batch_size, model.config.start_n_top * model.config.end_n_top)		)
         self.parent.assertEqual(result.cls_logits.shape						, (self.batch_size,)		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						int						, __lowerCAmelCase					:						str						, __lowerCAmelCase					:						List[Any]						, __lowerCAmelCase					:						Union[str, Any]						, __lowerCAmelCase					:						int						, __lowerCAmelCase					:						List[str]						, __lowerCAmelCase					:						List[Any]						, __lowerCAmelCase					:						Any						, __lowerCAmelCase					:						Dict						, __lowerCAmelCase					:						str						, ):






         """simple docstring"""






         _lowerCamelCase						:    Dict         =			FlaubertForSequenceClassification(__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()

         _lowerCamelCase						:    str         =			model(__lowerCAmelCase		)
         _lowerCamelCase						:    Tuple         =			model(__lowerCAmelCase						, labels=__lowerCAmelCase		)

         self.parent.assertEqual(result.loss.shape						, ()		)
         self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.type_sequence_label_size)		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Union[str, Any]						, __lowerCAmelCase					:						Any						, __lowerCAmelCase					:						List[Any]						, __lowerCAmelCase					:						Optional[int]						, __lowerCAmelCase					:						List[str]						, __lowerCAmelCase					:						List[str]						, __lowerCAmelCase					:						Dict						, __lowerCAmelCase					:						List[str]						, __lowerCAmelCase					:						Tuple						, __lowerCAmelCase					:						Union[str, Any]						, ):






         """simple docstring"""






         _lowerCamelCase						:    Any         =			self.num_labels
         _lowerCamelCase						:    List[str]         =			FlaubertForTokenClassification(__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()

         _lowerCamelCase						:    Union[str, Any]         =			model(__lowerCAmelCase						, attention_mask=__lowerCAmelCase						, labels=__lowerCAmelCase		)
         self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.seq_length, self.num_labels)		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Union[str, Any]						, __lowerCAmelCase					:						Any						, __lowerCAmelCase					:						List[str]						, __lowerCAmelCase					:						Dict						, __lowerCAmelCase					:						List[Any]						, __lowerCAmelCase					:						Union[str, Any]						, __lowerCAmelCase					:						List[Any]						, __lowerCAmelCase					:						Dict						, __lowerCAmelCase					:						Dict						, __lowerCAmelCase					:						Optional[Any]						, ):






         """simple docstring"""






         _lowerCamelCase						:    List[str]         =			self.num_choices
         _lowerCamelCase						:    Any         =			FlaubertForMultipleChoice(config=__lowerCAmelCase		)
         model.to(__lowerCAmelCase		)
         model.eval()
         _lowerCamelCase						:    List[str]         =			input_ids.unsqueeze(1		).expand(-1						, self.num_choices						, -1		).contiguous()
         _lowerCamelCase						:    List[str]         =			token_type_ids.unsqueeze(1		).expand(-1						, self.num_choices						, -1		).contiguous()
         _lowerCamelCase						:    List[Any]         =			input_mask.unsqueeze(1		).expand(-1						, self.num_choices						, -1		).contiguous()
         _lowerCamelCase						:    int         =			model(
             __lowerCAmelCase						, attention_mask=__lowerCAmelCase						, token_type_ids=__lowerCAmelCase						, labels=__lowerCAmelCase						, )
         self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.num_choices)		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Union[str, Any]		):






         """simple docstring"""






         _lowerCamelCase						:    Any         =			self.prepare_config_and_inputs()
         (
             (
             _lowerCamelCase
         )      ,   (
             _lowerCamelCase
         )      ,   (
             _lowerCamelCase
         )      ,   (
             _lowerCamelCase
         )      ,   (
             _lowerCamelCase
         )      ,   (
             _lowerCamelCase
         )      ,   (
             _lowerCamelCase
         )      ,   (
             _lowerCamelCase
         )      ,   (
             _lowerCamelCase
         )      ,   
         )						:    Optional[Any]         =			config_and_inputs
         _lowerCamelCase						:    int         =			{
             '''input_ids''': input_ids,
             '''token_type_ids''': token_type_ids,
             '''lengths''': input_lengths,
             '''attention_mask''': input_mask,
         }
         return config, inputs_dict






@require_torch
class 	__snake_case	(     _lowercase  ,					_lowercase  ,					unittest.TestCase):
  snake_case__    :       List[str]        =						(
      (
          FlaubertModel,
          FlaubertWithLMHeadModel,
          FlaubertForQuestionAnswering,
          FlaubertForQuestionAnsweringSimple,
          FlaubertForSequenceClassification,
          FlaubertForTokenClassification,
          FlaubertForMultipleChoice,
      )
      if is_torch_available()
      else ()
  )
  snake_case__    :       List[Any]        =						(
      {
          "feature-extraction": FlaubertModel,
          "fill-mask": FlaubertWithLMHeadModel,
          "question-answering": FlaubertForQuestionAnsweringSimple,
          "text-classification": FlaubertForSequenceClassification,
          "token-classification": FlaubertForTokenClassification,
          "zero-shot": FlaubertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )




  def 						SCREAMING_SNAKE_CASE    (			self					:						str						, __lowerCAmelCase					:						List[str]						, __lowerCAmelCase					:						int						, __lowerCAmelCase					:						Dict						, __lowerCAmelCase					:						Optional[int]						, __lowerCAmelCase					:						Tuple		):






         """simple docstring"""






         if (
             pipeline_test_casse_name == "QAPipelineTests"
             and tokenizer_name is not None
             and not tokenizer_name.endswith('''Fast'''		)
         ):
                # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
                # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
                # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
                return True

         return False




  def 						SCREAMING_SNAKE_CASE    (			self					:						Dict						, __lowerCAmelCase					:						int						, __lowerCAmelCase					:						int						, __lowerCAmelCase					:						Optional[int]=False		):






         """simple docstring"""






         _lowerCamelCase						:    Dict         =			super()._prepare_for_class(__lowerCAmelCase						, __lowerCAmelCase						, return_labels=__lowerCAmelCase		)

         if return_labels:
                if model_class.__name__ == "FlaubertForQuestionAnswering":
                       _lowerCamelCase						:    Dict         =			torch.zeros(
                           self.model_tester.batch_size						, dtype=torch.long						, device=__lowerCAmelCase		)
                       _lowerCamelCase						:    Any         =			torch.zeros(
                           self.model_tester.batch_size						, dtype=torch.long						, device=__lowerCAmelCase		)

         return inputs_dict




  def 						SCREAMING_SNAKE_CASE    (			self					:						List[Any]		):






         """simple docstring"""






         _lowerCamelCase						:    int         =			FlaubertModelTester(self		)
         _lowerCamelCase						:    str         =			ConfigTester(self						, config_class=__lowerCAmelCase						, emb_dim=3_7		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						List[Any]		):






         """simple docstring"""






         self.config_tester.run_common_tests()




  def 						SCREAMING_SNAKE_CASE    (			self					:						int		):






         """simple docstring"""






         _lowerCamelCase						:    Dict         =			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_model(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						List[str]		):






         """simple docstring"""






         _lowerCamelCase						:    Dict         =			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_lm_head(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Tuple		):






         """simple docstring"""






         _lowerCamelCase						:    int         =			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_simple_qa(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Tuple		):






         """simple docstring"""






         _lowerCamelCase						:    Tuple         =			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_qa(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Optional[int]		):






         """simple docstring"""






         _lowerCamelCase						:    Optional[int]         =			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_sequence_classif(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Dict		):






         """simple docstring"""






         _lowerCamelCase						:    int         =			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_token_classif(*__lowerCAmelCase		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						List[Any]		):






         """simple docstring"""






         _lowerCamelCase						:    List[Any]         =			self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_flaubert_multiple_choice(*__lowerCAmelCase		)




  @slow
  def 						SCREAMING_SNAKE_CASE    (			self					:						Union[str, Any]		):






         """simple docstring"""






         for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                _lowerCamelCase						:    Dict         =			FlaubertModel.from_pretrained(__lowerCAmelCase		)
                self.assertIsNotNone(__lowerCAmelCase		)




  @slow
  @require_torch_gpu
  def 						SCREAMING_SNAKE_CASE    (			self					:						Union[str, Any]		):






         """simple docstring"""






         _lowerCamelCase      ,   _lowerCamelCase						:    str         =			self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes:
                # FlauBertForMultipleChoice behaves incorrectly in JIT environments.
                if model_class == FlaubertForMultipleChoice:
                       return

                _lowerCamelCase						:    Any         =			True
                _lowerCamelCase						:    int         =			model_class(config=__lowerCAmelCase		)

                _lowerCamelCase						:    List[str]         =			self._prepare_for_class(__lowerCAmelCase						, __lowerCAmelCase		)
                _lowerCamelCase						:    int         =			torch.jit.trace(
                    __lowerCAmelCase						, (inputs_dict['''input_ids'''].to('''cpu'''		), inputs_dict['''attention_mask'''].to('''cpu'''		))		)

                with tempfile.TemporaryDirectory() as tmp:
                       torch.jit.save(__lowerCAmelCase						, os.path.join(__lowerCAmelCase						, '''traced_model.pt'''		)		)
                       _lowerCamelCase						:    Union[str, Any]         =			torch.jit.load(os.path.join(__lowerCAmelCase						, '''traced_model.pt'''		)						, map_location=__lowerCAmelCase		)
                       loaded(inputs_dict['''input_ids'''].to(__lowerCAmelCase		)						, inputs_dict['''attention_mask'''].to(__lowerCAmelCase		)		)






@require_torch
class 	__snake_case	(     unittest.TestCase):




  @slow
  def 						SCREAMING_SNAKE_CASE    (			self					:						Optional[int]		):






         """simple docstring"""






         _lowerCamelCase						:    Union[str, Any]         =			FlaubertModel.from_pretrained('''flaubert/flaubert_base_cased'''		)
         _lowerCamelCase						:    Any         =			torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]		)
         with torch.no_grad():
                _lowerCamelCase						:    Any         =			model(__lowerCAmelCase		)[0]
         _lowerCamelCase						:    Optional[Any]         =			torch.Size((1, 1_1, 7_6_8)		)
         self.assertEqual(output.shape						, __lowerCAmelCase		)
         _lowerCamelCase						:    Optional[int]         =			torch.tensor(
             [[[-2.62_51, -1.42_98, -0.02_27], [-2.85_10, -1.63_87, 0.22_58], [-2.81_14, -1.18_32, -0.30_66]]]		)

         self.assertTrue(torch.allclose(output[:, :3, :3]						, __lowerCAmelCase						, atol=1E-4		)		)


