



import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			_snake_case     (   lowercase__      ,		unittest.TestCase):
				UpperCamelCase__			:   Dict       						=LEDTokenizer
				UpperCamelCase__			:   List[str]       						=LEDTokenizerFast
				UpperCamelCase__			:   Union[str, Any]       						=True




				def 							A__ (  self   :  List[Any]  ):
							super().setUp()
							lowercase__   					=			[
							    "l",
							    "o",
							    "w",
							    "e",
							    "r",
							    "s",
							    "t",
							    "i",
							    "d",
							    "n",
							    "\u0120",
							    "\u0120l",
							    "\u0120n",
							    "\u0120lo",
							    "\u0120low",
							    "er",
							    "\u0120lowest",
							    "\u0120newer",
							    "\u0120wider",
							    "<unk>",
							]
							lowercase__   					=			dict(zip(__lowercase,   range(len(__lowercase  )  )  )  )
							lowercase__   					=			["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
							lowercase__   					=			{"unk_token": "<unk>"}

							lowercase__   					=			os.path.join(self.tmpdirname,   VOCAB_FILES_NAMES["vocab_file"]  )
							lowercase__   					=			os.path.join(self.tmpdirname,   VOCAB_FILES_NAMES["merges_file"]  )
							with open(self.vocab_file,   "w",   encoding="utf-8"  ) as fp:
										fp.write(json.dumps(__lowercase  ) + "\n"  )
							with open(self.merges_file,   "w",   encoding="utf-8"  ) as fp:
										fp.write("\n".join(__lowercase  )  )




				def 							A__ (  self   :  Any,   **__lowercase   :  Union[str, Any]  ):
							kwargs.update(self.special_tokens_map  )
							return self.tokenizer_class.from_pretrained(self.tmpdirname,   **__lowercase  )




				def 							A__ (  self   :  Dict,   **__lowercase   :  List[Any]  ):
							kwargs.update(self.special_tokens_map  )
							return self.rust_tokenizer_class.from_pretrained(self.tmpdirname,   **__lowercase  )




				def 							A__ (  self   :  str,   __lowercase   :  List[Any]  ):
							return "lower newer", "lower newer"




				@cached_property
				def 							A__ (  self   :  Tuple  ):
							return LEDTokenizer.from_pretrained("allenai/led-base-16384"  )




				@cached_property
				def 							A__ (  self   :  List[Any]  ):
							return LEDTokenizerFast.from_pretrained("allenai/led-base-16384"  )




				@require_torch
				def 							A__ (  self   :  Any  ):
							lowercase__   					=			["A long paragraph for summarization.", "Another paragraph for summarization."]
							lowercase__   					=			[0, 250, 251, 1_7818, 13, 3_9186, 1938, 4, 2]

							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										lowercase__   					=			tokenizer(__lowercase,   max_length=len(__lowercase  ),   padding=__lowercase,   return_tensors="pt"  )
										self.assertIsInstance(__lowercase,   __lowercase  )

										self.assertEqual((2, 9),   batch.input_ids.shape  )
										self.assertEqual((2, 9),   batch.attention_mask.shape  )
										lowercase__   					=			batch.input_ids.tolist()[0]
										self.assertListEqual(__lowercase,   __lowercase  )




				@require_torch
				def 							A__ (  self   :  Optional[int]  ):
							lowercase__   					=			["A long paragraph for summarization.", "Another paragraph for summarization."]
							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										lowercase__   					=			tokenizer(__lowercase,   padding=__lowercase,   return_tensors="pt"  )
										self.assertIn("input_ids",   __lowercase  )
										self.assertIn("attention_mask",   __lowercase  )
										self.assertNotIn("labels",   __lowercase  )
										self.assertNotIn("decoder_attention_mask",   __lowercase  )




				@require_torch
				def 							A__ (  self   :  str  ):
							lowercase__   					=			[
							    "Summary of the text.",
							    "Another summary.",
							]
							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										lowercase__   					=			tokenizer(text_target=__lowercase,   max_length=32,   padding="max_length",   return_tensors="pt"  )
										self.assertEqual(32,   targets["input_ids"].shape[1]  )




				@require_torch
				def 							A__ (  self   :  Union[str, Any]  ):
							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										lowercase__   					=			tokenizer(
										    ["I am a small frog" * 1024, "I am a small frog"],   padding=__lowercase,   truncation=__lowercase,   return_tensors="pt"  )
										self.assertIsInstance(__lowercase,   __lowercase  )
										self.assertEqual(batch.input_ids.shape,   (2, 5122)  )




				@require_torch
				def 							A__ (  self   :  Any  ):
							lowercase__   					=			["A long paragraph for summarization."]
							lowercase__   					=			[
							    "Summary of the text.",
							]
							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										lowercase__   					=			tokenizer(__lowercase,   return_tensors="pt"  )
										lowercase__   					=			tokenizer(text_target=__lowercase,   return_tensors="pt"  )
										lowercase__   					=			inputs["input_ids"]
										lowercase__   					=			targets["input_ids"]
										self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()  )
										self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()  )
										self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()  )
										self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()  )




				@require_torch
				def 							A__ (  self   :  Any  ):
							for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										lowercase__   					=			["Summary of the text.", "Another summary."]
										lowercase__   					=			[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

										lowercase__   					=			tokenizer(__lowercase,   padding=__lowercase  )
										lowercase__   					=			[[0] * len(__lowercase  ) for x in encoded_output["input_ids"]]
										lowercase__   					=			tokenizer.pad(__lowercase  )
										self.assertSequenceEqual(outputs["global_attention_mask"],   __lowercase  )




				def 							A__ (  self   :  Optional[int]  ):
							pass




				def 							A__ (  self   :  Tuple  ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
													lowercase__   					=			self.rust_tokenizer_class.from_pretrained(__lowercase,   **__lowercase  )
													lowercase__   					=			self.tokenizer_class.from_pretrained(__lowercase,   **__lowercase  )
													lowercase__   					=			"A, <mask> AllenNLP sentence."
													lowercase__   					=			tokenizer_r.encode_plus(__lowercase,   add_special_tokens=__lowercase,   return_token_type_ids=__lowercase  )
													lowercase__   					=			tokenizer_p.encode_plus(__lowercase,   add_special_tokens=__lowercase,   return_token_type_ids=__lowercase  )
													self.assertEqual(sum(tokens_r["token_type_ids"]  ),   sum(tokens_p["token_type_ids"]  )  )
													self.assertEqual(
													    sum(tokens_r["attention_mask"]  ) / len(tokens_r["attention_mask"]  ),   sum(tokens_p["attention_mask"]  ) / len(tokens_p["attention_mask"]  ),   )

													lowercase__   					=			tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]  )
													lowercase__   					=			tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]  )
													self.assertSequenceEqual(tokens_p["input_ids"],   [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]  )
													self.assertSequenceEqual(tokens_r["input_ids"],   [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]  )

													self.assertSequenceEqual(
													    __lowercase,   ["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]  )
													self.assertSequenceEqual(
													    __lowercase,   ["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]  )









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_						=       logging.get_logger(__name__)

lowercase_						=       {
    """facebook/dpr-ctx_encoder-single-nq-base""": (
        """https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/config.json"""
    ),
    """facebook/dpr-question_encoder-single-nq-base""": (
        """https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/config.json"""
    ),
    """facebook/dpr-reader-single-nq-base""": (
        """https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/config.json"""
    ),
    """facebook/dpr-ctx_encoder-multiset-base""": (
        """https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/config.json"""
    ),
    """facebook/dpr-question_encoder-multiset-base""": (
        """https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/config.json"""
    ),
    """facebook/dpr-reader-multiset-base""": (
        """https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/config.json"""
    ),
}
class 			_snake_case     (   lowercase__):
				UpperCamelCase__			:   str       						="""dpr"""




				def __init__(  self   :  int,   __lowercase   :  List[str]=3_0522,   __lowercase   :  Any=768,   __lowercase   :  Union[str, Any]=12,   __lowercase   :  Optional[int]=12,   __lowercase   :  List[Any]=3072,   __lowercase   :  List[str]="gelu",   __lowercase   :  Union[str, Any]=0.1,   __lowercase   :  str=0.1,   __lowercase   :  List[str]=512,   __lowercase   :  Optional[int]=2,   __lowercase   :  Dict=0.02,   __lowercase   :  Any=1e-1_2,   __lowercase   :  List[Any]=0,   __lowercase   :  str="absolute",   __lowercase   :  int = 0,   **__lowercase   :  str,   ):
							super().__init__(pad_token_id=__lowercase,   **__lowercase  )

							lowercase__   					=			vocab_size
							lowercase__   					=			hidden_size
							lowercase__   					=			num_hidden_layers
							lowercase__   					=			num_attention_heads
							lowercase__   					=			hidden_act
							lowercase__   					=			intermediate_size
							lowercase__   					=			hidden_dropout_prob
							lowercase__   					=			attention_probs_dropout_prob
							lowercase__   					=			max_position_embeddings
							lowercase__   					=			type_vocab_size
							lowercase__   					=			initializer_range
							lowercase__   					=			layer_norm_eps
							lowercase__   					=			projection_dim
							lowercase__   					=			position_embedding_type




