


'''simple docstring'''



import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


__UpperCAmelCase      :Optional[int]			    =       logging.getLogger(__name__)



class      a    :


     """simple docstring"""






     def __init__(    self					:		List[str]      )						->							Optional[int]:
          __UpperCAmelCase     :			str  							=      False

     def 			lowerCamelCase__			(    self					:		Union[str, Any]							,			snake_case					:		List[Any]							,			snake_case					:		List[str]							,			snake_case					:		List[str]							,			snake_case					:		str      )						->							Tuple:
          if not self.initialized:
               __UpperCAmelCase     :			Tuple  							=      RagRetriever(
                   snake_case							,			question_encoder_tokenizer=snake_case							,			generator_tokenizer=snake_case							,			index=snake_case							,			init_retrieval=snake_case							,			)
               __UpperCAmelCase     :			List[Any]  							=      True

     def 			lowerCamelCase__			(    self					:		int      )						->							Optional[Any]:
          self.retriever.index.init_index()






     def 			lowerCamelCase__			(    self					:		List[Any]							,			snake_case					:		Tuple							,			snake_case					:		Any      )						->							Optional[Any]:
          __UpperCAmelCase     :			Tuple  							=      self.retriever._main_retrieve(snake_case							,			snake_case      )
          return doc_ids, retrieved_doc_embeds



class      a    (		_a	):


     """simple docstring"""






     def __init__(    self					:		List[Any]							,			snake_case					:		Any							,			snake_case					:		Union[str, Any]							,			snake_case					:		Dict							,			snake_case					:		List[str]							,			snake_case					:		Optional[Any]=None      )						->							Any:
          if index is not None and index.is_initialized() and len(snake_case      ) > 0:
               raise ValueError(
                   '''When using Ray for distributed fine-tuning, '''
                   '''you\'ll need to provide the paths instead, '''
                   '''as the dataset and the index are loaded '''
                   '''separately. More info in examples/rag/use_own_knowledge_dataset.py '''      )
          super().__init__(
              snake_case							,			question_encoder_tokenizer=snake_case							,			generator_tokenizer=snake_case							,			index=snake_case							,			init_retrieval=snake_case							,			)
          __UpperCAmelCase     :			Any  							=      retrieval_workers
          if len(self.retrieval_workers      ) > 0:
               ray.get(
                   [
                       worker.create_rag_retriever.remote(snake_case							,			snake_case							,			snake_case							,			snake_case      )
                       for worker in self.retrieval_workers
                   ]      )

     def 			lowerCamelCase__			(    self					:		Union[str, Any]      )						->							Optional[int]:
          logger.info('''initializing retrieval'''      )

          if len(self.retrieval_workers      ) > 0:
               ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]      )
          else:
               # Non-distributed training. Load index into this same process.
               self.index.init_index()

     def 			lowerCamelCase__			(    self					:		Any							,			snake_case					:		Any							,			snake_case					:		Union[str, Any]      )						->							List[Any]:
          if len(self.retrieval_workers      ) > 0:
               # Select a random retrieval actor.
               __UpperCAmelCase     :			List[Any]  							=      self.retrieval_workers[random.randint(0							,			len(self.retrieval_workers      ) - 1      )]
               __UpperCAmelCase     :			List[Any]  							=      ray.get(random_worker.retrieve.remote(snake_case							,			snake_case      )      )
          else:
               __UpperCAmelCase     :			Tuple  							=      self._main_retrieve(snake_case							,			snake_case      )
          return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(snake_case      )

     @classmethod
     def 			lowerCamelCase__			(    cls					:		List[Any]							,			snake_case					:		Optional[Any]							,			snake_case					:		int=None							,			**snake_case					:		Tuple      )						->							int:
          return super(snake_case							,			cls      ).get_tokenizers(snake_case							,			snake_case							,			**snake_case      )






     @classmethod
     def 			lowerCamelCase__			(    cls					:		int							,			snake_case					:		List[str]							,			snake_case					:		Union[str, Any]							,			snake_case					:		List[Any]=None							,			**snake_case					:		Any      )						->							Dict:
          __UpperCAmelCase     :			str  							=      kwargs.pop('''config'''							,			snake_case      ) or RagConfig.from_pretrained(snake_case							,			**snake_case      )
          __UpperCAmelCase     :			List[str]  							=      RagTokenizer.from_pretrained(snake_case							,			config=snake_case      )
          __UpperCAmelCase     :			int  							=      rag_tokenizer.question_encoder
          __UpperCAmelCase     :			str  							=      rag_tokenizer.generator
          if indexed_dataset is not None:
               __UpperCAmelCase     :			List[Any]  							=      '''custom'''
               __UpperCAmelCase     :			Any  							=      CustomHFIndex(config.retrieval_vector_size							,			snake_case      )
          else:
               __UpperCAmelCase     :			Any  							=      cls._build_index(snake_case      )
          return cls(
              snake_case							,			question_encoder_tokenizer=snake_case							,			generator_tokenizer=snake_case							,			retrieval_workers=snake_case							,			index=snake_case							,			)



'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__UpperCAmelCase      :List[Any]			    =       {
    "configuration_lxmert": ["LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "LxmertConfig"],
    "tokenization_lxmert": ["LxmertTokenizer"],
}

try:
      if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase      :List[Any]			    =       ["LxmertTokenizerFast"]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase      :List[Any]			    =       [
          "LxmertEncoder",
          "LxmertForPreTraining",
          "LxmertForQuestionAnswering",
          "LxmertModel",
          "LxmertPreTrainedModel",
          "LxmertVisualFeatureEncoder",
          "LxmertXLayer",
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase      :List[str]			    =       [
          "TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST",
          "TFLxmertForPreTraining",
          "TFLxmertMainLayer",
          "TFLxmertModel",
          "TFLxmertPreTrainedModel",
          "TFLxmertVisualFeatureEncoder",
      ]


if TYPE_CHECKING:
      from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
      from .tokenization_lxmert import LxmertTokenizer

      try:
            if not is_tokenizers_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_lxmert_fast import LxmertTokenizerFast

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_lxmert import (
                LxmertEncoder,
                LxmertForPreTraining,
                LxmertForQuestionAnswering,
                LxmertModel,
                LxmertPreTrainedModel,
                LxmertVisualFeatureEncoder,
                LxmertXLayer,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_lxmert import (
                TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                TFLxmertForPreTraining,
                TFLxmertMainLayer,
                TFLxmertModel,
                TFLxmertPreTrainedModel,
                TFLxmertVisualFeatureEncoder,
            )

else:
      import sys

      __UpperCAmelCase      :int			    =       _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)