


'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase							=logging.get_logger(__name__)

lowercase							={
    "transfo-xl-wt103": "https://huggingface.co/transfo-xl-wt103/resolve/main/config.json",
}






class  __magic_name__							(		__lowerCAmelCase    ):
       UpperCAmelCase										="transfo-xl"
       UpperCAmelCase										=["mems"]
       UpperCAmelCase										={
           "n_token": "vocab_size",
           "hidden_size": "d_model",
           "num_attention_heads": "n_head",
           "num_hidden_layers": "n_layer",
       }






       def __init__(      self  ,							snake_case=2_6_7_7_3_5  ,							snake_case=[2_0_0_0_0, 4_0_0_0_0, 2_0_0_0_0_0]  ,							snake_case=1_0_2_4  ,							snake_case=1_0_2_4  ,							snake_case=1_6  ,							snake_case=6_4  ,							snake_case=4_0_9_6  ,							snake_case=4  ,							snake_case=False  ,							snake_case=1_8  ,							snake_case=1_6_0_0  ,							snake_case=1_0_0_0  ,							snake_case=True  ,							snake_case=True  ,							snake_case=0  ,							snake_case=-1  ,							snake_case=True  ,							snake_case=0.1  ,							snake_case=0.0  ,							snake_case=True  ,							snake_case="normal"  ,							snake_case=0.01  ,							snake_case=0.01  ,							snake_case=0.02  ,							snake_case=1E-5  ,							snake_case=0  ,							**snake_case  ,							)						->  Any:




          '''simple docstring'''
          _UpperCAmelCase       :       Any							   =vocab_size
          _UpperCAmelCase       :       List[Any]							   =[]
          self.cutoffs.extend(lowerCamelCase__)
          if proj_share_all_but_first:
             _UpperCAmelCase       :       List[str]							   =[False] + [True] * len(self.cutoffs)
          else:
             _UpperCAmelCase       :       str							   =[False] + [False] * len(self.cutoffs)
          _UpperCAmelCase       :       Optional[Any]							   =d_model
          _UpperCAmelCase       :       Any							   =d_embed
          _UpperCAmelCase       :       Tuple							   =d_head
          _UpperCAmelCase       :       Tuple							   =d_inner
          _UpperCAmelCase       :       Tuple							   =div_val
          _UpperCAmelCase       :       Optional[Any]							   =pre_lnorm
          _UpperCAmelCase       :       Tuple							   =n_layer
          _UpperCAmelCase       :       Optional[int]							   =n_head
          _UpperCAmelCase       :       List[Any]							   =mem_len
          _UpperCAmelCase       :       int							   =same_length
          _UpperCAmelCase       :       Optional[int]							   =attn_type
          _UpperCAmelCase       :       int							   =clamp_len
          _UpperCAmelCase       :       List[str]							   =sample_softmax
          _UpperCAmelCase       :       Optional[Any]							   =adaptive
          _UpperCAmelCase       :       str							   =dropout
          _UpperCAmelCase       :       Optional[int]							   =dropatt
          _UpperCAmelCase       :       Dict							   =untie_r
          _UpperCAmelCase       :       str							   =init
          _UpperCAmelCase       :       List[Any]							   =init_range
          _UpperCAmelCase       :       Dict							   =proj_init_std
          _UpperCAmelCase       :       int							   =init_std
          _UpperCAmelCase       :       Any							   =layer_norm_epsilon
          super().__init__(eos_token_id=lowerCamelCase__  ,							**lowerCamelCase__)






       @property
       def       lowerCAmelCase		(      self)						->  List[str]:




          '''simple docstring'''
          logger.info(f"The model {self.model_type} is one of the few models that has no sequence length limit.")
          return -1






       @max_position_embeddings.setter
       def       lowerCAmelCase		(      self  ,							snake_case)						->  Optional[int]:




          '''simple docstring'''
          raise NotImplementedError(
              f"The model {self.model_type} is one of the few models that has no sequence length limit.")





'''simple docstring'''






import logging
import os
from typing import List, Tuple

import numpy as np
import psutil
import torch
import torch.distributed as dist

from transformers import RagRetriever


lowercase							=logging.getLogger(__name__)






class  __magic_name__							(		lowerCAmelCase    ):






       def __init__(      self  ,							snake_case  ,							snake_case  ,							snake_case  ,							snake_case=None)						->  List[Any]:




          '''simple docstring'''
          super().__init__(
              snake_case  ,							question_encoder_tokenizer=snake_case  ,							generator_tokenizer=snake_case  ,							index=snake_case  ,							init_retrieval=snake_case  ,							)
          _UpperCAmelCase       :       Union[str, Any]							   =None






       def       lowerCAmelCase		(      self  ,							snake_case)						->  Optional[Any]:




          '''simple docstring'''
          logger.info('initializing retrieval')

          # initializing a separate process group for retrieval as the default
          # nccl backend doesn't support gather/scatter operations while gloo
          # is too slow to replace nccl for the core gpu communication
          if dist.is_initialized():
             logger.info('dist initialized')
             # needs to be set manually
             _UpperCAmelCase       :       Optional[Any]							   =self._infer_socket_ifname()
             # avoid clash with the NCCL port
             _UpperCAmelCase       :       Optional[int]							   =str(distributed_port + 1)
             _UpperCAmelCase       :       Any							   =dist.new_group(ranks=snake_case  ,							backend='gloo')

          # initialize retriever only on the main worker
          if not dist.is_initialized() or self._is_main():
             logger.info('dist not initialized / main')
             self.index.init_index()

          # all processes wait untill the retriever is initialized by the main process
          if dist.is_initialized():
             torch.distributed.barrier(group=self.process_group)






       def       lowerCAmelCase		(      self)						->  Tuple:




          '''simple docstring'''
          return dist.get_rank(group=self.process_group) == 0






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case  ,							snake_case=torch.floataa)						->  Union[str, Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   =torch.empty(snake_case  ,							dtype=snake_case)
          dist.scatter(snake_case  ,							src=0  ,							scatter_list=snake_case  ,							group=self.process_group)
          return target_tensor






       def       lowerCAmelCase		(      self)						->  Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   =psutil.net_if_addrs()
          # a hacky way to deal with varying network interface names
          _UpperCAmelCase       :       str							   =next((addr for addr in addrs if addr.startswith('e'))  ,							snake_case)
          return ifname






       def       lowerCAmelCase		(      self  ,							snake_case  ,							snake_case)						->  Tuple[np.ndarray, List[dict]]:




          '''simple docstring'''

          # single GPU training
          if not dist.is_initialized():
             _UpperCAmelCase   ,	_UpperCAmelCase       :       Optional[Any]							   =self._main_retrieve(snake_case  ,							snake_case)
             return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(snake_case)

          # distributed training
          _UpperCAmelCase       :       Optional[int]							   =dist.get_world_size(group=self.process_group)

          # gather logic
          _UpperCAmelCase       :       str							   =None
          if self._is_main():
             _UpperCAmelCase       :       Union[str, Any]							   =[torch.empty(question_hidden_states.shape  ,							dtype=torch.floataa) for _ in range(snake_case)]
          dist.gather(torch.tensor(snake_case)  ,							dst=0  ,							gather_list=snake_case  ,							group=self.process_group)

          # scatter logic
          _UpperCAmelCase       :       Optional[Any]							   =question_hidden_states.shape[0]
          _UpperCAmelCase       :       List[Any]							   =[]
          _UpperCAmelCase       :       Any							   =[]
          if self._is_main():
             assert len(snake_case) == world_size
             _UpperCAmelCase   ,	_UpperCAmelCase       :       Tuple							   =self._main_retrieve(torch.cat(snake_case).numpy()  ,							snake_case)
             _UpperCAmelCase   ,	_UpperCAmelCase       :       Union[str, Any]							   =torch.tensor(snake_case), torch.tensor(snake_case)
             _UpperCAmelCase       :       List[str]							   =self._chunk_tensor(snake_case  ,							snake_case)
             _UpperCAmelCase       :       Union[str, Any]							   =self._chunk_tensor(snake_case  ,							snake_case)
          _UpperCAmelCase       :       int							   =self._scattered(snake_case  ,							[n_queries, n_docs]  ,							target_type=torch.intaa)
          _UpperCAmelCase       :       Dict							   =self._scattered(snake_case  ,							[n_queries, n_docs, question_hidden_states.shape[1]])

          return retrieved_doc_embeds.numpy(), doc_ids.numpy(), self.index.get_doc_dicts(snake_case)

