




import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert import BertTokenizer


lowerCamelCase    :		Optional[Any]      =       logging.get_logger(__name__)

lowerCamelCase    :		List[Any]      =       {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowerCamelCase    :		Optional[Any]      =       {
    "vocab_file": {
        "facebook/dpr-ctx_encoder-single-nq-base": (
            "https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt"
        ),
        "facebook/dpr-ctx_encoder-multiset-base": (
            "https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "facebook/dpr-ctx_encoder-single-nq-base": (
            "https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json"
        ),
        "facebook/dpr-ctx_encoder-multiset-base": (
            "https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json"
        ),
    },
}
lowerCamelCase    :		Any      =       {
    "vocab_file": {
        "facebook/dpr-question_encoder-single-nq-base": (
            "https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt"
        ),
        "facebook/dpr-question_encoder-multiset-base": (
            "https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "facebook/dpr-question_encoder-single-nq-base": (
            "https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json"
        ),
        "facebook/dpr-question_encoder-multiset-base": (
            "https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json"
        ),
    },
}
lowerCamelCase    :		Optional[Any]      =       {
    "vocab_file": {
        "facebook/dpr-reader-single-nq-base": (
            "https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt"
        ),
        "facebook/dpr-reader-multiset-base": (
            "https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "facebook/dpr-reader-single-nq-base": (
            "https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json"
        ),
        "facebook/dpr-reader-multiset-base": (
            "https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json"
        ),
    },
}

lowerCamelCase    :		Optional[int]      =       {
    "facebook/dpr-ctx_encoder-single-nq-base": 512,
    "facebook/dpr-ctx_encoder-multiset-base": 512,
}
lowerCamelCase    :		Optional[int]      =       {
    "facebook/dpr-question_encoder-single-nq-base": 512,
    "facebook/dpr-question_encoder-multiset-base": 512,
}
lowerCamelCase    :		Tuple      =       {
    "facebook/dpr-reader-single-nq-base": 512,
    "facebook/dpr-reader-multiset-base": 512,
}


lowerCamelCase    :		List[Any]      =       {
    "facebook/dpr-ctx_encoder-single-nq-base": {"do_lower_case": True},
    "facebook/dpr-ctx_encoder-multiset-base": {"do_lower_case": True},
}
lowerCamelCase    :		Dict      =       {
    "facebook/dpr-question_encoder-single-nq-base": {"do_lower_case": True},
    "facebook/dpr-question_encoder-multiset-base": {"do_lower_case": True},
}
lowerCamelCase    :		List[str]      =       {
    "facebook/dpr-reader-single-nq-base": {"do_lower_case": True},
    "facebook/dpr-reader-multiset-base": {"do_lower_case": True},
}







class   A(						_UpperCAmelCase       ):



 '''simple docstring'''


 UpperCamelCase						       =						VOCAB_FILES_NAMES
 UpperCamelCase						       =						CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase						       =						CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase						       =						CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION







class   A(						_UpperCAmelCase       ):



 '''simple docstring'''


 UpperCamelCase						       =						VOCAB_FILES_NAMES
 UpperCamelCase						       =						QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase						       =						QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase						       =						QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION

lowerCamelCase    :		Optional[int]      =       collections.namedtuple(
    "DPRSpanPrediction", ["span_score", "relevance_score", "doc_id", "start_index", "end_index", "text"]
)

lowerCamelCase    :		Union[str, Any]      =       collections.namedtuple("DPRReaderOutput", ["start_logits", "end_logits", "relevance_logits"])


lowerCamelCase    :		int      =       r"\n    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.\n    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),\n    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`\n    with the format:\n\n    ```\n    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>\n    ```\n\n    Args:\n        questions (`str` or `List[str]`):\n            The questions to be encoded. You can specify one question for many passages. In this case, the question\n            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in\n            `titles` or `texts`.\n        titles (`str` or `List[str]`):\n            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.\n        texts (`str` or `List[str]`):\n            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.\n        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):\n            Activates and controls padding. Accepts the following values:\n\n            - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence\n              if provided).\n            - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided.\n            - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different\n              lengths).\n        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):\n            Activates and controls truncation. Accepts the following values:\n\n            - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or to\n              the maximum acceptable input length for the model if that argument is not provided. This will truncate\n              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch\n              of pairs) is provided.\n            - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the first\n              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the\n              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths\n              greater than the model maximum admissible input size).\n        max_length (`int`, *optional*):\n                Controls the maximum length to use by one of the truncation/padding parameters.\n\n                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length\n                is required by one of the truncation/padding parameters. If the model has no specific maximum input\n                length (like XLNet) truncation/padding to a maximum length will be deactivated.\n        return_tensors (`str` or [`~utils.TensorType`], *optional*):\n                If set, will return tensors instead of list of python integers. Acceptable values are:\n\n                - `'tf'`: Return TensorFlow `tf.constant` objects.\n                - `'pt'`: Return PyTorch `torch.Tensor` objects.\n                - `'np'`: Return Numpy `np.ndarray` objects.\n        return_attention_mask (`bool`, *optional*):\n            Whether or not to return the attention mask. If not set, will return the attention mask according to the\n            specific tokenizer's default, defined by the `return_outputs` attribute.\n\n            [What are attention masks?](../glossary#attention-mask)\n\n    Returns:\n        `Dict[str, List[List[int]]]`: A dictionary with the following keys:\n\n        - `input_ids`: List of token ids to be fed to a model.\n        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.\n    "







@add_start_docstrings(_UpperCAmelCase       )
class   A:



 '''simple docstring'''


 def __call__(    self :  int						,							A_ :  int						,							A_ :  Optional[str] = None						,							A_ :  Optional[str] = None						,							A_ :  Union[bool, str] = False						,							A_ :  Union[bool, str] = False						,							A_ :  Optional[int] = None						,							A_ :  Optional[Union[str, TensorType]] = None						,							A_ :  Optional[bool] = None						,							**A_ :  int						,							)       ->     int:

       """simple docstring"""






       if titles is None and texts is None:
             return super().__call__(
                 lowercase_						,							padding=lowercase_						,							truncation=lowercase_						,							max_length=lowercase_						,							return_tensors=lowercase_						,							return_attention_mask=lowercase_						,							**lowercase_						,							)
       elif titles is None or texts is None:
             lowerCamelCase_							=   titles if texts is None else texts
             return super().__call__(
                 lowercase_						,							lowercase_						,							padding=lowercase_						,							truncation=lowercase_						,							max_length=lowercase_						,							return_tensors=lowercase_						,							return_attention_mask=lowercase_						,							**lowercase_						,							)
       lowerCamelCase_							=   titles if not isinstance(lowercase_						,							lowercase_  ) else [titles]
       lowerCamelCase_							=   texts if not isinstance(lowercase_						,							lowercase_  ) else [texts]
       lowerCamelCase_							=   len(lowercase_  )
       lowerCamelCase_							=   questions if not isinstance(lowercase_						,							lowercase_  ) else [questions] * n_passages
       if len(lowercase_  ) != len(lowercase_  ):
             raise ValueError(
                 f"""There should be as many titles than texts but got {len(lowercase_  )} titles and {len(lowercase_  )} texts."""  )
       lowerCamelCase_							=   super().__call__(lowercase_						,							lowercase_						,							padding=lowercase_						,							truncation=lowercase_  )["""input_ids"""]
       lowerCamelCase_							=   super().__call__(lowercase_						,							add_special_tokens=lowercase_						,							padding=lowercase_						,							truncation=lowercase_  )["""input_ids"""]
       lowerCamelCase_							=   {
           """input_ids""": [
               (encoded_question_and_title + encoded_text)[:max_length]
               if max_length is not None and truncation
               else encoded_question_and_title + encoded_text
               for encoded_question_and_title, encoded_text in zip(lowercase_						,							lowercase_  )
           ]
       }
       if return_attention_mask is not False:
             lowerCamelCase_							=   []
             for input_ids in encoded_inputs["input_ids"]:
                   attention_mask.append([int(input_id != self.pad_token_id  ) for input_id in input_ids]  )
             lowerCamelCase_							=   attention_mask
       return self.pad(lowercase_						,							padding=lowercase_						,							max_length=lowercase_						,							return_tensors=lowercase_  )

 def     a__			(    self :  Union[str, Any]						,							A_ :  BatchEncoding						,							A_ :  DPRReaderOutput						,							A_ :  int = 16						,							A_ :  int = 64						,							A_ :  int = 4						,							)       ->     Dict:

       """simple docstring"""






       lowerCamelCase_							=   reader_input["""input_ids"""]
       lowerCamelCase_							=   reader_output[:3]
       lowerCamelCase_							=   len(lowercase_  )
       lowerCamelCase_							=   sorted(range(lowercase_  )						,							reverse=lowercase_						,							key=relevance_logits.__getitem__  )
       lowerCamelCase_							=   []
       for doc_id in sorted_docs:
             lowerCamelCase_							=   list(input_ids[doc_id]  )
             # assuming question & title information is at the beginning of the sequence
             lowerCamelCase_							=   sequence_ids.index(self.sep_token_id						,							2  ) + 1  # second sep id
             if sequence_ids[-1] == self.pad_token_id:
                   lowerCamelCase_							=   sequence_ids.index(self.pad_token_id  )
             else:
                   lowerCamelCase_							=   len(lowercase_  )

             lowerCamelCase_							=   self._get_best_spans(
                 start_logits=start_logits[doc_id][passage_offset:sequence_len]						,							end_logits=end_logits[doc_id][passage_offset:sequence_len]						,							max_answer_length=lowercase_						,							top_spans=lowercase_						,							)
             for start_index, end_index in best_spans:
                   start_index += passage_offset
                   end_index += passage_offset
                   nbest_spans_predictions.append(
                       DPRSpanPrediction(
                           span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]						,							relevance_score=relevance_logits[doc_id]						,							doc_id=lowercase_						,							start_index=lowercase_						,							end_index=lowercase_						,							text=self.decode(sequence_ids[start_index : end_index + 1]  )						,							)  )
             if len(lowercase_  ) >= num_spans:
                   break
       return nbest_spans_predictions[:num_spans]





 def     a__			(    self :  Dict						,							A_ :  List[int]						,							A_ :  List[int]						,							A_ :  int						,							A_ :  int						,							)       ->     Optional[Any]:

       """simple docstring"""






       lowerCamelCase_							=   []
       for start_index, start_score in enumerate(lowercase_  ):
             for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]  ):
                   scores.append(((start_index, start_index + answer_length), start_score + end_score)  )
       lowerCamelCase_							=   sorted(lowercase_						,							key=lambda A_  : x[1]						,							reverse=lowercase_  )
       lowerCamelCase_							=   []
       for (start_index, end_index), score in scores:
             if start_index > end_index:
                   raise ValueError(f"""Wrong span indices: [{start_index}:{end_index}]"""  )
             lowerCamelCase_							=   end_index - start_index + 1
             if length > max_answer_length:
                   raise ValueError(f"""Span is too long: {length} > {max_answer_length}"""  )
             if any(
                 start_index <= prev_start_index <= prev_end_index <= end_index
                 or prev_start_index <= start_index <= end_index <= prev_end_index
                 for (prev_start_index, prev_end_index) in chosen_span_intervals  ):
                   continue
             chosen_span_intervals.append((start_index, end_index)  )

             if len(lowercase_  ) == top_spans:
                   break
       return chosen_span_intervals







@add_end_docstrings(_UpperCAmelCase       )
class   A(						_UpperCAmelCase							,				_UpperCAmelCase       ):



 '''simple docstring'''


 UpperCamelCase						       =						VOCAB_FILES_NAMES
 UpperCamelCase						       =						READER_PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase						       =						READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase						       =						READER_PRETRAINED_INIT_CONFIGURATION
 UpperCamelCase						       =						['''input_ids''', '''attention_mask''']









import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("1.0.0a"):
 raise Exception("requires fairseq >= 1.0.0a")

logging.set_verbosity_info()
lowerCamelCase    :		Tuple			   =       logging.get_logger(__name__)

lowerCamelCase    :		Union[str, Any]			   =       "Hello world! cécé herlolip"

def       _SCREAMING_SNAKE_CASE      (							lowercase     :	str     ,							lowercase     :	str     ,							lowercase     :	bool					):


      '''simple docstring'''



      lowerCamelCase_							=   FairseqRobertaModel.from_pretrained(lowercase					)
      roberta.eval()  # disable dropout
      lowerCamelCase_							=   roberta.model.encoder.sentence_encoder
      lowerCamelCase_							=   XLMRobertaConfig(
          vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings     ,							hidden_size=roberta.cfg.model.encoder_embed_dim     ,							num_hidden_layers=roberta.cfg.model.encoder_layers     ,							num_attention_heads=roberta.cfg.model.encoder_attention_heads     ,							intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim     ,							max_position_embeddings=5_14     ,							type_vocab_size=1     ,							layer_norm_eps=1e-5     ,							)
      if classification_head:
            lowerCamelCase_							=   roberta.model.classification_heads['mnli'].out_proj.weight.shape[0]

      print('Our RoBERTa config:'     ,							lowercase					)

      lowerCamelCase_							=   XLMRobertaXLForSequenceClassification(lowercase					) if classification_head else XLMRobertaXLForMaskedLM(lowercase					)
      model.eval()

      # Now let's copy all the weights.
      # Embeddings
      lowerCamelCase_							=   roberta_sent_encoder.embed_tokens.weight
      lowerCamelCase_							=   roberta_sent_encoder.embed_positions.weight
      lowerCamelCase_							=   torch.zeros_like(
          model.roberta.embeddings.token_type_embeddings.weight					)  # just zero them out b/c RoBERTa doesn't use them.

      lowerCamelCase_							=   roberta_sent_encoder.layer_norm.weight
      lowerCamelCase_							=   roberta_sent_encoder.layer_norm.bias

      for i in range(config.num_hidden_layers					):
            # Encoder: start of layer
            lowerCamelCase_							=   model.roberta.encoder.layer[i]
            lowerCamelCase_							=   roberta_sent_encoder.layers[i]

            lowerCamelCase_							=   layer.attention
            lowerCamelCase_							=   roberta_layer.self_attn_layer_norm.weight
            lowerCamelCase_							=   roberta_layer.self_attn_layer_norm.bias

            # self attention
            lowerCamelCase_							=   layer.attention.self
            assert (
                roberta_layer.self_attn.k_proj.weight.data.shape
                == roberta_layer.self_attn.q_proj.weight.data.shape
                == roberta_layer.self_attn.v_proj.weight.data.shape
                == torch.Size((config.hidden_size, config.hidden_size)					)
            )

            lowerCamelCase_							=   roberta_layer.self_attn.q_proj.weight
            lowerCamelCase_							=   roberta_layer.self_attn.q_proj.bias
            lowerCamelCase_							=   roberta_layer.self_attn.k_proj.weight
            lowerCamelCase_							=   roberta_layer.self_attn.k_proj.bias
            lowerCamelCase_							=   roberta_layer.self_attn.v_proj.weight
            lowerCamelCase_							=   roberta_layer.self_attn.v_proj.bias

            # self-attention output
            lowerCamelCase_							=   layer.attention.output
            assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
            lowerCamelCase_							=   roberta_layer.self_attn.out_proj.weight
            lowerCamelCase_							=   roberta_layer.self_attn.out_proj.bias

            # this one is final layer norm
            lowerCamelCase_							=   roberta_layer.final_layer_norm.weight
            lowerCamelCase_							=   roberta_layer.final_layer_norm.bias

            # intermediate
            lowerCamelCase_							=   layer.intermediate
            assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
            lowerCamelCase_							=   roberta_layer.fca.weight
            lowerCamelCase_							=   roberta_layer.fca.bias

            # output
            lowerCamelCase_							=   layer.output
            assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
            lowerCamelCase_							=   roberta_layer.fca.weight
            lowerCamelCase_							=   roberta_layer.fca.bias
            # end of layer

      if classification_head:
            lowerCamelCase_							=   roberta.model.classification_heads['mnli'].dense.weight
            lowerCamelCase_							=   roberta.model.classification_heads['mnli'].dense.bias
            lowerCamelCase_							=   roberta.model.classification_heads['mnli'].out_proj.weight
            lowerCamelCase_							=   roberta.model.classification_heads['mnli'].out_proj.bias
      else:
            # LM Head
            lowerCamelCase_							=   roberta.model.encoder.lm_head.dense.weight
            lowerCamelCase_							=   roberta.model.encoder.lm_head.dense.bias
            lowerCamelCase_							=   roberta.model.encoder.lm_head.layer_norm.weight
            lowerCamelCase_							=   roberta.model.encoder.lm_head.layer_norm.bias
            lowerCamelCase_							=   roberta.model.encoder.lm_head.weight
            lowerCamelCase_							=   roberta.model.encoder.lm_head.bias

      # Let's check that we get the same results.
      lowerCamelCase_							=   roberta.encode(lowercase					).unsqueeze(0					)  # batch of size 1

      lowerCamelCase_							=   model(lowercase					)[0]
      if classification_head:
            lowerCamelCase_							=   roberta.model.classification_heads['mnli'](roberta.extract_features(lowercase					)					)
      else:
            lowerCamelCase_							=   roberta.model(lowercase					)[0]
      print(our_output.shape     ,							their_output.shape					)
      lowerCamelCase_							=   torch.max(torch.abs(our_output - their_output					)					).item()
      print(f"""max_absolute_diff = {max_absolute_diff}"""					)  # ~ 1e-7
      lowerCamelCase_							=   torch.allclose(lowercase     ,							lowercase     ,							atol=1e-3					)
      print('Do both models output the same tensors?'     ,							'🔥' if success else '💩'					)
      if not success:
            raise Exception('Something went wRoNg'					)

      pathlib.Path(lowercase					).mkdir(parents=lowercase     ,							exist_ok=lowercase					)
      print(f"""Saving model to {pytorch_dump_folder_path}"""					)
      model.save_pretrained(lowercase					)


if __name__ == "__main__":
 lowerCamelCase    :		Optional[Any]			   =       argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     "--roberta_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
 )
 parser.add_argument(
     "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
 )
 parser.add_argument(
     "--classification_head", action="store_true", help="Whether to convert a final classification head."
 )
 lowerCamelCase    :		List[Any]			   =       parser.parse_args()
 convert_xlm_roberta_xl_checkpoint_to_pytorch(
     args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
 )



