



import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_  :	str 				=					logging.get_logger(__name__)


lowerCamelCase_  :	List[Any] 				=					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all BART models at https://huggingface.co/models?filter=bart
lowerCamelCase_  :	List[Any] 				=					{
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
}

lowerCamelCase_  :	Optional[int] 				=					{
    """facebook/bart-base""": 10_24,
    """facebook/bart-large""": 10_24,
    """facebook/bart-large-mnli""": 10_24,
    """facebook/bart-large-cnn""": 10_24,
    """facebook/bart-large-xsum""": 10_24,
    """yjernite/bart_eli5""": 10_24,
}







@lru_cache()
def     A__   (  )					->		Dict:
				UpperCamelCase_:							Dict		=  (
				    list(range(ord("""!"""  )       ,      ord("""~"""  ) + 1  )  ) + list(range(ord("""¡"""  )       ,      ord("""¬"""  ) + 1  )  ) + list(range(ord("""®"""  )       ,      ord("""ÿ"""  ) + 1  )  )
				)
				UpperCamelCase_:							Optional[int]		=  bs[:]
				UpperCamelCase_:							List[Any]		=  0
				for b in range(2**8  ):
								if b not in bs:
												bs.append(lowerCamelCase  )
												cs.append(2**8 + n  )
												n += 1
				UpperCamelCase_:							Dict		=  [chr(lowerCamelCase  ) for n in cs]
				return dict(zip(lowerCamelCase       ,      lowerCamelCase  )  )







def     A__   (  lowerCamelCase  )					->		int:
				UpperCamelCase_:							int		=  set()
				UpperCamelCase_:							Optional[int]		=  word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)  )
								UpperCamelCase_:							Tuple		=  char
				return pairs





class     _UpperCamelCase			(   _A					):






			'''simple docstring'''

			__UpperCamelCase :  str    =	VOCAB_FILES_NAMES
			__UpperCamelCase :  Optional[int]    =	PRETRAINED_VOCAB_FILES_MAP
			__UpperCamelCase :  List[Any]    =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__UpperCamelCase :  Any    =	["""input_ids""", """attention_mask"""]

			def __init__(    self	:    Tuple  ,		snake_case_	:    Tuple  ,		snake_case_	:    List[str]  ,		snake_case_	:    str="replace"  ,		snake_case_	:    Any="<s>"  ,		snake_case_	:    Optional[int]="</s>"  ,		snake_case_	:    Union[str, Any]="</s>"  ,		snake_case_	:    List[Any]="<s>"  ,		snake_case_	:    List[Any]="<unk>"  ,		snake_case_	:    Any="<pad>"  ,		snake_case_	:    Union[str, Any]="<mask>"  ,		snake_case_	:    Union[str, Any]=False  ,		**snake_case_	:    Optional[Any]  ,		):
							UpperCamelCase_:							Optional[int]		=  AddedToken(snake_case_  ,		lstrip=snake_case_  ,		rstrip=snake_case_ ) if isinstance(snake_case_  ,		snake_case_ ) else bos_token
							UpperCamelCase_:							List[Any]		=  AddedToken(snake_case_  ,		lstrip=snake_case_  ,		rstrip=snake_case_ ) if isinstance(snake_case_  ,		snake_case_ ) else eos_token
							UpperCamelCase_:							Tuple		=  AddedToken(snake_case_  ,		lstrip=snake_case_  ,		rstrip=snake_case_ ) if isinstance(snake_case_  ,		snake_case_ ) else sep_token
							UpperCamelCase_:							Optional[Any]		=  AddedToken(snake_case_  ,		lstrip=snake_case_  ,		rstrip=snake_case_ ) if isinstance(snake_case_  ,		snake_case_ ) else cls_token
							UpperCamelCase_:							Any		=  AddedToken(snake_case_  ,		lstrip=snake_case_  ,		rstrip=snake_case_ ) if isinstance(snake_case_  ,		snake_case_ ) else unk_token
							UpperCamelCase_:							Tuple		=  AddedToken(snake_case_  ,		lstrip=snake_case_  ,		rstrip=snake_case_ ) if isinstance(snake_case_  ,		snake_case_ ) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							UpperCamelCase_:							Union[str, Any]		=  AddedToken(snake_case_  ,		lstrip=snake_case_  ,		rstrip=snake_case_ ) if isinstance(snake_case_  ,		snake_case_ ) else mask_token

							super().__init__(
							    errors=snake_case_  ,		bos_token=snake_case_  ,		eos_token=snake_case_  ,		unk_token=snake_case_  ,		sep_token=snake_case_  ,		cls_token=snake_case_  ,		pad_token=snake_case_  ,		mask_token=snake_case_  ,		add_prefix_space=snake_case_  ,		**snake_case_  ,		)

							with open(snake_case_  ,		encoding="""utf-8""" ) as vocab_handle:
											UpperCamelCase_:							str		=  json.load(snake_case_ )
							UpperCamelCase_:							Tuple		=  {v: k for k, v in self.encoder.items()}
							UpperCamelCase_:							List[Any]		=  errors  # how to handle errors in decoding
							UpperCamelCase_:							List[str]		=  bytes_to_unicode()
							UpperCamelCase_:							int		=  {v: k for k, v in self.byte_encoder.items()}
							with open(snake_case_  ,		encoding="""utf-8""" ) as merges_handle:
											UpperCamelCase_:							Optional[int]		=  merges_handle.read().split("""\n""" )[1:-1]
							UpperCamelCase_:							Any		=  [tuple(merge.split() ) for merge in bpe_merges]
							UpperCamelCase_:							List[Any]		=  dict(zip(snake_case_  ,		range(len(snake_case_ ) ) ) )
							UpperCamelCase_:							List[str]		=  {}
							UpperCamelCase_:							int		=  add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							UpperCamelCase_:							Any		=  re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" )

			@property
			def  lowerCAmelCase__		(    self	:    Optional[Any] ):
							return len(self.encoder )

			def  lowerCAmelCase__		(    self	:    int ):
							return dict(self.encoder  ,		**self.added_tokens_encoder )

			def  lowerCAmelCase__		(    self	:    List[str]  ,		snake_case_	:    List[str] ):
							if token in self.cache:
											return self.cache[token]
							UpperCamelCase_:							Optional[int]		=  tuple(snake_case_ )
							UpperCamelCase_:							List[Any]		=  get_pairs(snake_case_ )

							if not pairs:
											return token

							while True:
											UpperCamelCase_:							Any		=  min(snake_case_  ,		key=lambda snake_case_ : self.bpe_ranks.get(snake_case_  ,		float("""inf""" ) ) )
											if bigram not in self.bpe_ranks:
															break
											UpperCamelCase_:							int		=  bigram
											UpperCamelCase_:							Optional[Any]		=  []
											UpperCamelCase_:							Dict		=  0
											while i < len(snake_case_ ):
															try:
																			UpperCamelCase_:							Union[str, Any]		=  word.index(snake_case_  ,		snake_case_ )
															except ValueError:
																			new_word.extend(word[i:] )
																			break
															else:
																			new_word.extend(word[i:j] )
																			UpperCamelCase_:							Optional[Any]		=  j

															if word[i] == first and i < len(snake_case_ ) - 1 and word[i + 1] == second:
																			new_word.append(first + second )
																			i += 2
															else:
																			new_word.append(word[i] )
																			i += 1
											UpperCamelCase_:							Any		=  tuple(snake_case_ )
											UpperCamelCase_:							Tuple		=  new_word
											if len(snake_case_ ) == 1:
															break
											else:
															UpperCamelCase_:							Dict		=  get_pairs(snake_case_ )
							UpperCamelCase_:							Optional[Any]		=  """ """.join(snake_case_ )
							UpperCamelCase_:							Optional[int]		=  word
							return word

			def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    int ):
							UpperCamelCase_:							Tuple		=  []
							for token in re.findall(self.pat  ,		snake_case_ ):
											UpperCamelCase_:							Union[str, Any]		=  """""".join(
											    self.byte_encoder[b] for b in token.encode("""utf-8""" ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case_ ).split(""" """ ) )
							return bpe_tokens

			def  lowerCAmelCase__		(    self	:    List[str]  ,		snake_case_	:    Dict ):
							return self.encoder.get(snake_case_  ,		self.encoder.get(self.unk_token ) )

			def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    str ):
							return self.decoder.get(snake_case_ )

			def  lowerCAmelCase__		(    self	:    Any  ,		snake_case_	:    List[str] ):
							UpperCamelCase_:							str		=  """""".join(snake_case_ )
							UpperCamelCase_:							Tuple		=  bytearray([self.byte_decoder[c] for c in text] ).decode("""utf-8"""  ,		errors=self.errors )
							return text

			def  lowerCAmelCase__		(    self	:    str  ,		snake_case_	:    str  ,		snake_case_	:    Optional[str] = None ):
							if not os.path.isdir(snake_case_ ):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
											return
							UpperCamelCase_:							Optional[Any]		=  os.path.join(
							    snake_case_  ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )
							UpperCamelCase_:							int		=  os.path.join(
							    snake_case_  ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""] )

							with open(snake_case_  ,		"""w"""  ,		encoding="""utf-8""" ) as f:
											f.write(json.dumps(self.encoder  ,		indent=2  ,		sort_keys=snake_case_  ,		ensure_ascii=snake_case_ ) + """\n""" )

							UpperCamelCase_:							List[Any]		=  0
							with open(snake_case_  ,		"""w"""  ,		encoding="""utf-8""" ) as writer:
											writer.write("""#version: 0.2\n""" )
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,		key=lambda snake_case_ : kv[1] ):
															if index != token_index:
																			logger.warning(
																			    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																			    """ Please check that the tokenizer is not corrupted!""" )
																			UpperCamelCase_:							Optional[int]		=  token_index
															writer.write(""" """.join(snake_case_ ) + """\n""" )
															index += 1

							return vocab_file, merge_file

			def  lowerCAmelCase__		(    self	:    Any  ,		snake_case_	:    List[int]  ,		snake_case_	:    Optional[List[int]] = None ):
							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							UpperCamelCase_:							Tuple		=  [self.cls_token_id]
							UpperCamelCase_:							Optional[Any]		=  [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep

			def  lowerCAmelCase__		(    self	:    Optional[int]  ,		snake_case_	:    List[int]  ,		snake_case_	:    Optional[List[int]] = None  ,		snake_case_	:    bool = False ):
							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=snake_case_  ,		token_ids_a=snake_case_  ,		already_has_special_tokens=snake_case_ )

							if token_ids_a is None:
											return [1] + ([0] * len(snake_case_ )) + [1]
							return [1] + ([0] * len(snake_case_ )) + [1, 1] + ([0] * len(snake_case_ )) + [1]

			def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    List[int]  ,		snake_case_	:    Optional[List[int]] = None ):
							UpperCamelCase_:							Optional[int]		=  [self.sep_token_id]
							UpperCamelCase_:							Optional[int]		=  [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]




			def  lowerCAmelCase__		(    self	:    str  ,		snake_case_	:    List[str]  ,		snake_case_	:    Any=False  ,		**snake_case_	:    Optional[Any] ):
							UpperCamelCase_:							Tuple		=  kwargs.pop("""add_prefix_space"""  ,		self.add_prefix_space )
							if (is_split_into_words or add_prefix_space) and (len(snake_case_ ) > 0 and not text[0].isspace()):
											UpperCamelCase_:							Tuple		=  """ """ + text
							return (text, kwargs)








import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowerCamelCase_  :	Optional[Any] 				=					datasets.utils.logging.get_logger(__name__)

@dataclass
class     _UpperCamelCase			(   datasets.BuilderConfig					):






			'''simple docstring'''

			__UpperCamelCase :  Optional[datasets.Features]    =	None
			__UpperCamelCase :  str    =	"utf-8"
			__UpperCamelCase :  Optional[str]    =	None
			__UpperCamelCase :  Optional[str]    =	None
			__UpperCamelCase :  bool    =	True  # deprecated
			__UpperCamelCase :  Optional[int]    =	None  # deprecated
			__UpperCamelCase :  int    =	10 << 20  # 10MB
			__UpperCamelCase :  Optional[bool]    =	None




class     _UpperCamelCase			(   datasets.ArrowBasedBuilder					):






			'''simple docstring'''

			__UpperCamelCase :  Tuple    =	JsonConfig

			def  lowerCAmelCase__		(    self	:    int ):
							if self.config.block_size is not None:
											logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead""" )
											UpperCamelCase_:							List[str]		=  self.config.block_size
							if self.config.use_threads is not True:
											logger.warning(
											    """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore.""" )
							if self.config.newlines_in_values is not None:
											raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported""" )
							return datasets.DatasetInfo(features=self.config.features )

			def  lowerCAmelCase__		(    self	:    Dict  ,		snake_case_	:    str ):
							if not self.config.data_files:
											raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}''' )
							UpperCamelCase_:							Dict		=  dl_manager.download_and_extract(self.config.data_files )
							if isinstance(snake_case_  ,		(str, list, tuple) ):
											UpperCamelCase_:							List[Any]		=  data_files
											if isinstance(snake_case_  ,		snake_case_ ):
															UpperCamelCase_:							str		=  [files]
											UpperCamelCase_:							Any		=  [dl_manager.iter_files(snake_case_ ) for file in files]
											return [datasets.SplitGenerator(name=datasets.Split.TRAIN  ,		gen_kwargs={"""files""": files} )]
							UpperCamelCase_:							Dict		=  []
							for split_name, files in data_files.items():
											if isinstance(snake_case_  ,		snake_case_ ):
															UpperCamelCase_:							Tuple		=  [files]
											UpperCamelCase_:							Optional[int]		=  [dl_manager.iter_files(snake_case_ ) for file in files]
											splits.append(datasets.SplitGenerator(name=snake_case_  ,		gen_kwargs={"""files""": files} ) )
							return splits

			def  lowerCAmelCase__		(    self	:    str  ,		snake_case_	:    pa.Table ):
							if self.config.features is not None:
											# adding missing columns
											for column_name in set(self.config.features ) - set(pa_table.column_names ):
															UpperCamelCase_:							Union[str, Any]		=  self.config.features.arrow_schema.field(snake_case_ ).type
															UpperCamelCase_:							Tuple		=  pa_table.append_column(snake_case_  ,		pa.array([None] * len(snake_case_ )  ,		type=snake_case_ ) )
											# more expensive cast to support nested structures with keys in a different order
											# allows str <-> int/float or str to Audio for example
											UpperCamelCase_:							int		=  table_cast(snake_case_  ,		self.config.features.arrow_schema )
							return pa_table




			def  lowerCAmelCase__		(    self	:    Dict  ,		snake_case_	:    Optional[Any] ):
							for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case_ ) ):
											# If the file is one json object and if we need to look at the list of items in one specific field
											if self.config.field is not None:
															with open(snake_case_  ,		encoding=self.config.encoding  ,		errors=self.config.encoding_errors ) as f:
																			UpperCamelCase_:							Dict		=  json.load(snake_case_ )

															# We keep only the field we are interested in
															UpperCamelCase_:							Optional[int]		=  dataset[self.config.field]

															# We accept two format: a list of dicts or a dict of lists
															if isinstance(snake_case_  ,		(list, tuple) ):
																			UpperCamelCase_:							Optional[int]		=  set().union(*[row.keys() for row in dataset] )
																			UpperCamelCase_:							int		=  {col: [row.get(snake_case_ ) for row in dataset] for col in keys}
															else:
																			UpperCamelCase_:							Optional[int]		=  dataset
															UpperCamelCase_:							List[str]		=  pa.Table.from_pydict(snake_case_ )
															yield file_idx, self._cast_table(snake_case_ )

											# If the file has one json object per line
											else:
															with open(snake_case_  ,		"""rb""" ) as f:
																			UpperCamelCase_:							Optional[int]		=  0
																			# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																			# Set a default minimum value of 16kB if the chunk size is really small
																			UpperCamelCase_:							Optional[int]		=  max(self.config.chunksize // 32  ,		16 << 10 )
																			UpperCamelCase_:							Tuple		=  (
																			    self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
																			)
																			while True:
																							UpperCamelCase_:							int		=  f.read(self.config.chunksize )
																							if not batch:
																											break
																							# Finish current line
																							try:
																											batch += f.readline()
																							except (AttributeError, io.UnsupportedOperation):
																											batch += readline(snake_case_ )
																							# PyArrow only accepts utf-8 encoded bytes
																							if self.config.encoding != "utf-8":
																											UpperCamelCase_:							Tuple		=  batch.decode(self.config.encoding  ,		errors=snake_case_ ).encode("""utf-8""" )
																							try:
																											while True:
																															try:
																																			UpperCamelCase_:							Tuple		=  paj.read_json(
																																			    io.BytesIO(snake_case_ )  ,		read_options=paj.ReadOptions(block_size=snake_case_ ) )
																																			break
																															except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																			if (
																																			    isinstance(snake_case_  ,		pa.ArrowInvalid )
																																			    and "straddling" not in str(snake_case_ )
																																			    or block_size > len(snake_case_ )
																																			):
																																							raise
																																			else:
																																							# Increase the block size in case it was too small.
																																							# The block size will be reset for the next file.
																																							logger.debug(
																																							    f'''Batch of {len(snake_case_ )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.''' )
																																							block_size *= 2
																							except pa.ArrowInvalid as e:
																											try:
																															with open(
																															    snake_case_  ,		encoding=self.config.encoding  ,		errors=self.config.encoding_errors ) as f:
																																			UpperCamelCase_:							Optional[Any]		=  json.load(snake_case_ )
																											except json.JSONDecodeError:
																															logger.error(f'''Failed to read file \'{file}\' with error {type(snake_case_ )}: {e}''' )
																															raise e
																											# If possible, parse the file as a list of json objects and exit the loop
																											if isinstance(snake_case_  ,		snake_case_ ):  # list is the only sequence type supported in JSON
																															try:
																																			UpperCamelCase_:							Any		=  set().union(*[row.keys() for row in dataset] )
																																			UpperCamelCase_:							List[str]		=  {col: [row.get(snake_case_ ) for row in dataset] for col in keys}
																																			UpperCamelCase_:							int		=  pa.Table.from_pydict(snake_case_ )
																															except (pa.ArrowInvalid, AttributeError) as e:
																																			logger.error(f'''Failed to read file \'{file}\' with error {type(snake_case_ )}: {e}''' )
																																			raise ValueError(f'''Not able to read records in the JSON file at {file}.''' ) from None
																															yield file_idx, self._cast_table(snake_case_ )
																															break
																											else:
																															logger.error(f'''Failed to read file \'{file}\' with error {type(snake_case_ )}: {e}''' )
																															raise ValueError(
																															    f'''Not able to read records in the JSON file at {file}. '''
																															    f'''You should probably indicate the field of the JSON file containing your records. '''
																															    f'''This JSON file contain the following fields: {str(list(dataset.keys() ) )}. '''
																															    f'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. ''' ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																							yield (file_idx, batch_idx), self._cast_table(snake_case_ )
																							batch_idx += 1






