


'''simple docstring'''






import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer






class  __magic_name__							(		lowerCAmelCase    ):






       def       lowerCAmelCase		(      self)						->  Tuple:




          '''simple docstring'''
          _UpperCAmelCase       :       str							   =tempfile.mkdtemp()
          _UpperCAmelCase       :       Optional[int]							   =5

          # Realm tok
          _UpperCAmelCase       :       Optional[Any]							   =[
              '[UNK]',
              '[CLS]',
              '[SEP]',
              '[PAD]',
              '[MASK]',
              'test',
              'question',
              'this',
              'is',
              'the',
              'first',
              'second',
              'third',
              'fourth',
              'fifth',
              'record',
              'want',
              '##want',
              '##ed',
              'wa',
              'un',
              'runn',
              '##ing',
              ',',
              'low',
              'lowest',
          ]
          _UpperCAmelCase       :       int							   =os.path.join(self.tmpdirname  ,							'realm_tokenizer')
          os.makedirs(snake_case  ,							exist_ok=snake_case)
          _UpperCAmelCase       :       List[str]							   =os.path.join(snake_case  ,							VOCAB_FILES_NAMES['vocab_file'])
          with open(self.vocab_file  ,							'w'  ,							encoding='utf-8') as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))

          _UpperCAmelCase       :       int							   =os.path.join(self.tmpdirname  ,							'realm_block_records')
          os.makedirs(snake_case  ,							exist_ok=snake_case)






       def       lowerCAmelCase		(      self)						->  RealmTokenizer:




          '''simple docstring'''
          return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,							'realm_tokenizer'))






       def       lowerCAmelCase		(      self)						->  Optional[Any]:




          '''simple docstring'''
          shutil.rmtree(self.tmpdirname)






       def       lowerCAmelCase		(      self)						->  Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase       :       Union[str, Any]							   =RealmConfig(num_block_records=self.num_block_records)
          return config






       def       lowerCAmelCase		(      self)						->  str:




          '''simple docstring'''
          _UpperCAmelCase       :       str							   =Dataset.from_dict(
              {
                  'id': ['0', '1'],
                  'question': ['foo', 'bar'],
                  'answers': [['Foo', 'Bar'], ['Bar']],
              })
          return dataset






       def       lowerCAmelCase		(      self)						->  Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       List[str]							   =np.array(
              [
                  B'This is the first record',
                  B'This is the second record',
                  B'This is the third record',
                  B'This is the fourth record',
                  B'This is the fifth record',
                  B'This is a longer longer longer record',
              ]  ,							dtype=snake_case  ,							)
          return block_records






       def       lowerCAmelCase		(      self)						->  Union[str, Any]:




          '''simple docstring'''
          _UpperCAmelCase       :       int							   =RealmRetriever(
              block_records=self.get_dummy_block_records()  ,							tokenizer=self.get_tokenizer()  ,							)
          return retriever






       def       lowerCAmelCase		(      self)						->  Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase       :       List[str]							   =self.get_config()
          _UpperCAmelCase       :       Optional[Any]							   =self.get_dummy_retriever()
          _UpperCAmelCase       :       str							   =retriever.tokenizer

          _UpperCAmelCase       :       Any							   =np.array([0, 3]  ,							dtype='long')
          _UpperCAmelCase       :       Any							   =tokenizer(['Test question']).input_ids
          _UpperCAmelCase       :       str							   =tokenizer(
              ['the fourth']  ,							add_special_tokens=snake_case  ,							return_token_type_ids=snake_case  ,							return_attention_mask=snake_case  ,							).input_ids
          _UpperCAmelCase       :       Dict							   =config.reader_seq_len

          _UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase       :       List[Any]							   =retriever(
              snake_case  ,							snake_case  ,							answer_ids=snake_case  ,							max_length=snake_case  ,							return_tensors='np')

          self.assertEqual(len(snake_case)  ,							2)
          self.assertEqual(len(snake_case)  ,							2)
          self.assertEqual(len(snake_case)  ,							2)
          self.assertEqual(concat_inputs.input_ids.shape  ,							(2, 1_0))
          self.assertEqual(concat_inputs.attention_mask.shape  ,							(2, 1_0))
          self.assertEqual(concat_inputs.token_type_ids.shape  ,							(2, 1_0))
          self.assertEqual(concat_inputs.special_tokens_mask.shape  ,							(2, 1_0))
          self.assertEqual(
              tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0])  ,							['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']  ,							)
          self.assertEqual(
              tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1])  ,							['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']  ,							)






       def       lowerCAmelCase		(      self)						->  str:




          '''simple docstring'''
          _UpperCAmelCase       :       Any							   =self.get_config()
          _UpperCAmelCase       :       Optional[int]							   =self.get_dummy_retriever()
          _UpperCAmelCase       :       List[str]							   =retriever.tokenizer

          _UpperCAmelCase       :       List[Any]							   =np.array([0, 3, 5]  ,							dtype='long')
          _UpperCAmelCase       :       Tuple							   =tokenizer(['Test question']).input_ids
          _UpperCAmelCase       :       Union[str, Any]							   =tokenizer(
              ['the fourth', 'longer longer']  ,							add_special_tokens=snake_case  ,							return_token_type_ids=snake_case  ,							return_attention_mask=snake_case  ,							).input_ids
          _UpperCAmelCase       :       Union[str, Any]							   =config.reader_seq_len

          _UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase   ,	_UpperCAmelCase       :       Optional[Any]							   =retriever(
              snake_case  ,							snake_case  ,							answer_ids=snake_case  ,							max_length=snake_case  ,							return_tensors='np')

          self.assertEqual([False, True, True]  ,							snake_case)
          self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]  ,							snake_case)
          self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]  ,							snake_case)






       def       lowerCAmelCase		(      self)						->  Any:




          '''simple docstring'''
          _UpperCAmelCase       :       Dict							   =self.get_dummy_retriever()
          retriever.save_pretrained(os.path.join(self.tmpdirname  ,							'realm_block_records'))

          # Test local path
          _UpperCAmelCase       :       Tuple							   =retriever.from_pretrained(os.path.join(self.tmpdirname  ,							'realm_block_records'))
          self.assertEqual(retriever.block_records[0]  ,							B'This is the first record')

          # Test mocked remote path
          with patch('transformers.models.realm.retrieval_realm.hf_hub_download') as mock_hf_hub_download:
             _UpperCAmelCase       :       Dict							   =os.path.join(
                 os.path.join(self.tmpdirname  ,							'realm_block_records')  ,							_REALM_BLOCK_RECORDS_FILENAME)
             _UpperCAmelCase       :       List[str]							   =RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa')

          self.assertEqual(retriever.block_records[0]  ,							B'This is the first record')





'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowercase							={
    'configuration_groupvit': [
        'GROUPVIT_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'GroupViTConfig',
        'GroupViTOnnxConfig',
        'GroupViTTextConfig',
        'GroupViTVisionConfig',
    ],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowercase							=[
     'GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST',
     'GroupViTModel',
     'GroupViTPreTrainedModel',
     'GroupViTTextModel',
     'GroupViTVisionModel',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowercase							=[
     'TF_GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST',
     'TFGroupViTModel',
     'TFGroupViTPreTrainedModel',
     'TFGroupViTTextModel',
     'TFGroupViTVisionModel',
 ]

if TYPE_CHECKING:
 from .configuration_groupvit import (
     GROUPVIT_PRETRAINED_CONFIG_ARCHIVE_MAP,
     GroupViTConfig,
     GroupViTOnnxConfig,
     GroupViTTextConfig,
     GroupViTVisionConfig,
 )

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_groupvit import (
      GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
      GroupViTModel,
      GroupViTPreTrainedModel,
      GroupViTTextModel,
      GroupViTVisionModel,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_groupvit import (
      TF_GROUPVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFGroupViTModel,
      TFGroupViTPreTrainedModel,
      TFGroupViTTextModel,
      TFGroupViTVisionModel,
  )

else:
 import sys

 lowercase							=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

