




import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer




class       a__	( UpperCamelCase__				):







     def       lowerCAmelCase_    (     self			)  ->   List[str]:



          '''simple docstring'''



          a     =				tempfile.mkdtemp()
          a     =				5

          # Realm tok
          a     =				[
              "[UNK]",
              "[CLS]",
              "[SEP]",
              "[PAD]",
              "[MASK]",
              "test",
              "question",
              "this",
              "is",
              "the",
              "first",
              "second",
              "third",
              "fourth",
              "fifth",
              "record",
              "want",
              "##want",
              "##ed",
              "wa",
              "un",
              "runn",
              "##ing",
              ",",
              "low",
              "lowest",
          ]
          a     =				os.path.join(self.tmpdirname  ,		"realm_tokenizer"			)
          os.makedirs(A  ,		exist_ok=A			)
          a     =				os.path.join(A  ,		VOCAB_FILES_NAMES["vocab_file"]			)
          with open(self.vocab_file  ,		"w"  ,		encoding="utf-8"			) as vocab_writer:
               vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)

          a     =				os.path.join(self.tmpdirname  ,		"realm_block_records"			)
          os.makedirs(A  ,		exist_ok=A			)







     def       lowerCAmelCase_    (     self			)  ->   RealmTokenizer:



          '''simple docstring'''



          return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,		"realm_tokenizer"			)			)







     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          shutil.rmtree(self.tmpdirname			)







     def       lowerCAmelCase_    (     self			)  ->   List[str]:



          '''simple docstring'''



          a     =				RealmConfig(num_block_records=self.num_block_records			)
          return config







     def       lowerCAmelCase_    (     self			)  ->   Any:



          '''simple docstring'''



          a     =				Dataset.from_dict(
              {
                  "id": ["0", "1"],
                  "question": ["foo", "bar"],
                  "answers": [["Foo", "Bar"], ["Bar"]],
              }			)
          return dataset







     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          a     =				np.array(
              [
                  b"This is the first record",
                  b"This is the second record",
                  b"This is the third record",
                  b"This is the fourth record",
                  b"This is the fifth record",
                  b"This is a longer longer longer record",
              ]  ,		dtype=A  ,		)
          return block_records







     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          a     =				RealmRetriever(
              block_records=self.get_dummy_block_records()  ,		tokenizer=self.get_tokenizer()  ,		)
          return retriever







     def       lowerCAmelCase_    (     self			)  ->   str:



          '''simple docstring'''



          a     =				self.get_config()
          a     =				self.get_dummy_retriever()
          a     =				retriever.tokenizer

          a     =				np.array([0, 3]  ,		dtype="long"			)
          a     =				tokenizer(["Test question"]			).input_ids
          a     =				tokenizer(
              ["the fourth"]  ,		add_special_tokens=A  ,		return_token_type_ids=A  ,		return_attention_mask=A  ,		).input_ids
          a     =				config.reader_seq_len

          a  ,				a  ,				a  ,				a     =				retriever(
              A  ,		A  ,		answer_ids=A  ,		max_length=A  ,		return_tensors="np"			)

          self.assertEqual(len(A			)  ,		2			)
          self.assertEqual(len(A			)  ,		2			)
          self.assertEqual(len(A			)  ,		2			)
          self.assertEqual(concat_inputs.input_ids.shape  ,		(2, 10)			)
          self.assertEqual(concat_inputs.attention_mask.shape  ,		(2, 10)			)
          self.assertEqual(concat_inputs.token_type_ids.shape  ,		(2, 10)			)
          self.assertEqual(concat_inputs.special_tokens_mask.shape  ,		(2, 10)			)
          self.assertEqual(
              tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]			)  ,		["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "first", "record", "[SEP]"]  ,		)
          self.assertEqual(
              tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]			)  ,		["[CLS]", "test", "question", "[SEP]", "this", "is", "the", "fourth", "record", "[SEP]"]  ,		)







     def       lowerCAmelCase_    (     self			)  ->   List[Any]:



          '''simple docstring'''



          a     =				self.get_config()
          a     =				self.get_dummy_retriever()
          a     =				retriever.tokenizer

          a     =				np.array([0, 3, 5]  ,		dtype="long"			)
          a     =				tokenizer(["Test question"]			).input_ids
          a     =				tokenizer(
              ["the fourth", "longer longer"]  ,		add_special_tokens=A  ,		return_token_type_ids=A  ,		return_attention_mask=A  ,		).input_ids
          a     =				config.reader_seq_len

          a  ,				a  ,				a  ,				a     =				retriever(
              A  ,		A  ,		answer_ids=A  ,		max_length=A  ,		return_tensors="np"			)

          self.assertEqual([False, True, True]  ,		A			)
          self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]  ,		A			)
          self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]  ,		A			)







     def       lowerCAmelCase_    (     self			)  ->   Dict:



          '''simple docstring'''



          a     =				self.get_dummy_retriever()
          retriever.save_pretrained(os.path.join(self.tmpdirname  ,		"realm_block_records"			)			)

          # Test local path
          a     =				retriever.from_pretrained(os.path.join(self.tmpdirname  ,		"realm_block_records"			)			)
          self.assertEqual(retriever.block_records[0]  ,		b"This is the first record"			)

          # Test mocked remote path
          with patch("transformers.models.realm.retrieval_realm.hf_hub_download"			) as mock_hf_hub_download:
               a     =				os.path.join(
                   os.path.join(self.tmpdirname  ,		"realm_block_records"			)  ,		_REALM_BLOCK_RECORDS_FILENAME			)
               a     =				RealmRetriever.from_pretrained("google/realm-cc-news-pretrained-openqa"			)

          self.assertEqual(retriever.block_records[0]  ,		b"This is the first record"			)







lowercase__				:      str	   = "\n# Installazione di Transformers\n! pip install transformers datasets\n# Per installare dalla fonte invece dell'ultima versione rilasciata, commenta il comando sopra e\n# rimuovi la modalità commento al comando seguente.\n# ! pip install git+https://github.com/huggingface/transformers.git\n"

lowercase__				:      Any	   = [{"type": "code", "content": INSTALL_CONTENT}]
lowercase__				:      Any	   = {
    "{processor_class}": "FakeProcessorClass",
    "{model_class}": "FakeModelClass",
    "{object_class}": "FakeObjectClass",
}

