
"""simple docstring"""

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_A						=			logging.get_logger(__name__)

_A						=			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_A						=			{
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

_A						=			{
    'google/realm-cc-news-pretrained-embedder': 5_1_2,
    'google/realm-cc-news-pretrained-encoder': 5_1_2,
    'google/realm-cc-news-pretrained-scorer': 5_1_2,
    'google/realm-cc-news-pretrained-openqa': 5_1_2,
    'google/realm-orqa-nq-openqa': 5_1_2,
    'google/realm-orqa-nq-reader': 5_1_2,
    'google/realm-orqa-wq-openqa': 5_1_2,
    'google/realm-orqa-wq-reader': 5_1_2,
}

_A						=			{
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}






class 	_lowercase       (    __UpperCAmelCase		):
	lowercase_				=     VOCAB_FILES_NAMES
	lowercase_				=     PRETRAINED_VOCAB_FILES_MAP
	lowercase_				=     PRETRAINED_INIT_CONFIGURATION
	lowercase_				=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase_				=     RealmTokenizer




	def __init__(			self		,			UpperCAmelCase_=None		,			UpperCAmelCase_=None		,			UpperCAmelCase_=True		,			UpperCAmelCase_="[UNK]"		,			UpperCAmelCase_="[SEP]"		,			UpperCAmelCase_="[PAD]"		,			UpperCAmelCase_="[CLS]"		,			UpperCAmelCase_="[MASK]"		,			UpperCAmelCase_=True		,			UpperCAmelCase_=None		,			**UpperCAmelCase_		,			)		->							str:
			super().__init__(
			    UpperCAmelCase_		,			tokenizer_file=UpperCAmelCase_		,			do_lower_case=UpperCAmelCase_		,			unk_token=UpperCAmelCase_		,			sep_token=UpperCAmelCase_		,			pad_token=UpperCAmelCase_		,			cls_token=UpperCAmelCase_		,			mask_token=UpperCAmelCase_		,			tokenize_chinese_chars=UpperCAmelCase_		,			strip_accents=UpperCAmelCase_		,			**UpperCAmelCase_		,			)

			lowerCamelCase					:      Tuple          =   json.loads(self.backend_tokenizer.normalizer.__getstate__() )
			if (
			    normalizer_state.get('lowercase'		,			UpperCAmelCase_ ) != do_lower_case
			    or normalizer_state.get('strip_accents'		,			UpperCAmelCase_ ) != strip_accents
			    or normalizer_state.get('handle_chinese_chars'		,			UpperCAmelCase_ ) != tokenize_chinese_chars
			):
					lowerCamelCase					:      Optional[int]          =   getattr(UpperCAmelCase_		,			normalizer_state.pop('type' ) )
					lowerCamelCase					:      Tuple          =   do_lower_case
					lowerCamelCase					:      Dict          =   strip_accents
					lowerCamelCase					:      Dict          =   tokenize_chinese_chars
					lowerCamelCase					:      Union[str, Any]          =   normalizer_class(**UpperCAmelCase_ )

			lowerCamelCase					:      str          =   do_lower_case




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			**UpperCAmelCase_ )		->							Optional[Any]:
			lowerCamelCase					:      List[str]          =   PaddingStrategy.MAX_LENGTH

			lowerCamelCase					:      int          =   text
			lowerCamelCase					:      List[Any]          =   kwargs.pop('text_pair'		,			UpperCAmelCase_ )
			lowerCamelCase					:      Optional[Any]          =   kwargs.pop('return_tensors'		,			UpperCAmelCase_ )

			lowerCamelCase					:      Any          =   {
			    'input_ids': [],
			    'attention_mask': [],
			    'token_type_ids': [],
			}

			for idx, candidate_text in enumerate(UpperCAmelCase_ ):
					if batch_text_pair is not None:
							lowerCamelCase					:      Any          =   batch_text_pair[idx]
					else:
							lowerCamelCase					:      Union[str, Any]          =   None

					lowerCamelCase					:      Any          =   super().__call__(UpperCAmelCase_		,			UpperCAmelCase_		,			return_tensors=UpperCAmelCase_		,			**UpperCAmelCase_ )

					lowerCamelCase					:      str          =   encoded_candidates.get('input_ids' )
					lowerCamelCase					:      List[str]          =   encoded_candidates.get('attention_mask' )
					lowerCamelCase					:      Any          =   encoded_candidates.get('token_type_ids' )

					if encoded_input_ids is not None:
							output_data["input_ids"].append(UpperCAmelCase_ )
					if encoded_attention_mask is not None:
							output_data["attention_mask"].append(UpperCAmelCase_ )
					if encoded_token_type_ids is not None:
							output_data["token_type_ids"].append(UpperCAmelCase_ )

			lowerCamelCase					:      Dict          =   {key: item for key, item in output_data.items() if len(UpperCAmelCase_ ) != 0}

			return BatchEncoding(UpperCAmelCase_		,			tensor_type=UpperCAmelCase_ )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_=None )		->							str:
			lowerCamelCase					:      int          =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

			if token_ids_a:
					output += token_ids_a + [self.sep_token_id]

			return output




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ = None )		->							List[int]:
			lowerCamelCase					:      Any          =   [self.sep_token_id]
			lowerCamelCase					:      Optional[int]          =   [self.cls_token_id]
			if token_ids_a is None:
					return len(cls + token_ids_a + sep ) * [0]
			return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ = None )		->							Tuple[str]:
			lowerCamelCase					:      Dict          =   self._tokenizer.model.save(UpperCAmelCase_		,			name=UpperCAmelCase_ )
			return tuple(UpperCAmelCase_ )








"""simple docstring"""

import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


_A						=			logging.getLogger(__name__)






class 	_lowercase       :




	def __init__(			self )		->							Tuple:
			lowerCamelCase					:      Dict          =   False




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_ )		->							Any:
			if not self.initialized:
					lowerCamelCase					:      Optional[int]          =   RagRetriever(
					    UpperCAmelCase_		,			question_encoder_tokenizer=UpperCAmelCase_		,			generator_tokenizer=UpperCAmelCase_		,			index=UpperCAmelCase_		,			init_retrieval=UpperCAmelCase_		,			)
					lowerCamelCase					:      Union[str, Any]          =   True




	def       _UpperCamelCase   (			self )		->							Union[str, Any]:
			self.retriever.index.init_index()




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ )		->							str:
			lowerCamelCase     ,   lowerCamelCase					:      int          =   self.retriever._main_retrieve(UpperCAmelCase_		,			UpperCAmelCase_ )
			return doc_ids, retrieved_doc_embeds






class 	_lowercase       (    __UpperCAmelCase		):




	def __init__(			self		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_=None )		->							Dict:
			if index is not None and index.is_initialized() and len(UpperCAmelCase_ ) > 0:
					raise ValueError(
					    'When using Ray for distributed fine-tuning, '
					    'you\'ll need to provide the paths instead, '
					    'as the dataset and the index are loaded '
					    'separately. More info in examples/rag/use_own_knowledge_dataset.py ' )
			super().__init__(
			    UpperCAmelCase_		,			question_encoder_tokenizer=UpperCAmelCase_		,			generator_tokenizer=UpperCAmelCase_		,			index=UpperCAmelCase_		,			init_retrieval=UpperCAmelCase_		,			)
			lowerCamelCase					:      List[str]          =   retrieval_workers
			if len(self.retrieval_workers ) > 0:
					ray.get(
					    [
					        worker.create_rag_retriever.remote(UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_ )
					        for worker in self.retrieval_workers
					    ] )




	def       _UpperCamelCase   (			self )		->							Dict:
			logger.info('initializing retrieval' )

			if len(self.retrieval_workers ) > 0:
					ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers] )
			else:
					# Non-distributed training. Load index into this same process.
					self.index.init_index()




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ )		->							List[Any]:
			if len(self.retrieval_workers ) > 0:
					# Select a random retrieval actor.
					lowerCamelCase					:      Tuple          =   self.retrieval_workers[random.randint(0		,			len(self.retrieval_workers ) - 1 )]
					lowerCamelCase     ,   lowerCamelCase					:      Tuple          =   ray.get(random_worker.retrieve.remote(UpperCAmelCase_		,			UpperCAmelCase_ ) )
			else:
					lowerCamelCase     ,   lowerCamelCase					:      int          =   self._main_retrieve(UpperCAmelCase_		,			UpperCAmelCase_ )
			return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(UpperCAmelCase_ )




	@classmethod
	def       _UpperCamelCase   (			cls		,			UpperCAmelCase_		,			UpperCAmelCase_=None		,			**UpperCAmelCase_ )		->							Dict:
			return super(UpperCAmelCase_		,			cls ).get_tokenizers(UpperCAmelCase_		,			UpperCAmelCase_		,			**UpperCAmelCase_ )




	@classmethod
	def       _UpperCamelCase   (			cls		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_=None		,			**UpperCAmelCase_ )		->							Dict:
			lowerCamelCase					:      str          =   kwargs.pop('config'		,			UpperCAmelCase_ ) or RagConfig.from_pretrained(UpperCAmelCase_		,			**UpperCAmelCase_ )
			lowerCamelCase					:      Any          =   RagTokenizer.from_pretrained(UpperCAmelCase_		,			config=UpperCAmelCase_ )
			lowerCamelCase					:      Any          =   rag_tokenizer.question_encoder
			lowerCamelCase					:      Union[str, Any]          =   rag_tokenizer.generator
			if indexed_dataset is not None:
					lowerCamelCase					:      int          =   'custom'
					lowerCamelCase					:      int          =   CustomHFIndex(config.retrieval_vector_size		,			UpperCAmelCase_ )
			else:
					lowerCamelCase					:      Optional[Any]          =   cls._build_index(UpperCAmelCase_ )
			return cls(
			    UpperCAmelCase_		,			question_encoder_tokenizer=UpperCAmelCase_		,			generator_tokenizer=UpperCAmelCase_		,			retrieval_workers=UpperCAmelCase_		,			index=UpperCAmelCase_		,			)






