



"""simple docstring"""





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase       	=						logging.get_logger(__name__)

lowerCamelCase       	=						"""▁"""

lowerCamelCase       	=						{"""vocab_file""": """sentencepiece.bpe.model""", """monolingual_vocab_file""": """dict.txt"""}

lowerCamelCase       	=						{
    """vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model""",
    },
    """monolingual_vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt""",
    },
}

lowerCamelCase       	=						{"""vinai/bartpho-syllable""": 1_024}
class 				lowercase__       (   SCREAMING_SNAKE_CASE				):







       '''simple docstring'''


       UpperCamelCase						=					VOCAB_FILES_NAMES
       UpperCamelCase						=					PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase						=					['''input_ids''', '''attention_mask''']



       def __init__(							self      :  Optional[int]		,    _UpperCAmelCase      :  Optional[int]		,    _UpperCAmelCase      :  Tuple		,    _UpperCAmelCase      :  Dict="<s>"		,    _UpperCAmelCase      :  List[str]="</s>"		,    _UpperCAmelCase      :  List[Any]="</s>"		,    _UpperCAmelCase      :  int="<s>"		,    _UpperCAmelCase      :  Tuple="<unk>"		,    _UpperCAmelCase      :  List[Any]="<pad>"		,    _UpperCAmelCase      :  Tuple="<mask>"		,    _UpperCAmelCase      :  Optional[Dict[str, Any]] = None		,    **_UpperCAmelCase      :  Dict		,    )			->						None:


             '''simple docstring'''




             UpperCAmelCase_      			=							AddedToken(_UpperCAmelCase		,    lstrip=_UpperCAmelCase		,    rstrip=_UpperCAmelCase   ) if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ) else mask_token

             UpperCAmelCase_      			=							{} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 bos_token=_UpperCAmelCase		,    eos_token=_UpperCAmelCase		,    unk_token=_UpperCAmelCase		,    sep_token=_UpperCAmelCase		,    cls_token=_UpperCAmelCase		,    pad_token=_UpperCAmelCase		,    mask_token=_UpperCAmelCase		,    sp_model_kwargs=self.sp_model_kwargs		,    **_UpperCAmelCase		,    )

             UpperCAmelCase_      			=							vocab_file
             UpperCAmelCase_      			=							monolingual_vocab_file
             UpperCAmelCase_      			=							spm.SentencePieceProcessor(**self.sp_model_kwargs   )
             self.sp_model.Load(str(_UpperCAmelCase   )   )

             # Load the reduced vocab

             # Keep order of special tokens for backward compatibility
             UpperCAmelCase_      			=							{}
             UpperCAmelCase_      			=							0
             for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
                   if str(_UpperCAmelCase   ) not in self.fairseq_tokens_to_ids:
                         UpperCAmelCase_      			=							cnt
                         cnt += 1
             with open(_UpperCAmelCase		,    "r"		,    encoding="utf-8"   ) as f:
                   for line in f.readlines():
                         UpperCAmelCase_      			=							line.strip().split()[0]
                         UpperCAmelCase_      			=							len(self.fairseq_tokens_to_ids   )
             if str(_UpperCAmelCase   ) not in self.fairseq_tokens_to_ids:
                   UpperCAmelCase_      			=							len(self.fairseq_tokens_to_ids   )

             UpperCAmelCase_      			=							{v: k for k, v in self.fairseq_tokens_to_ids.items()}



       def __getstate__(							self      :  int   )			->						List[str]:


             '''simple docstring'''




             UpperCAmelCase_      			=							self.__dict__.copy()
             UpperCAmelCase_      			=							None
             UpperCAmelCase_      			=							self.sp_model.serialized_model_proto()
             return state



       def __setstate__(							self      :  int		,    _UpperCAmelCase      :  Optional[int]   )			->						Dict:


             '''simple docstring'''




             UpperCAmelCase_      			=							d

             # for backward compatibility
             if not hasattr(self		,    "sp_model_kwargs"   ):
                   UpperCAmelCase_      			=							{}

             UpperCAmelCase_      			=							spm.SentencePieceProcessor(**self.sp_model_kwargs   )
             self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )



       def 					lowercase__    (							self      :  Union[str, Any]		,    _UpperCAmelCase      :  List[int]		,    _UpperCAmelCase      :  Optional[List[int]] = None   )			->						List[int]:


             '''simple docstring'''




             if token_ids_a is None:
                   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
             UpperCAmelCase_      			=							[self.cls_token_id]
             UpperCAmelCase_      			=							[self.sep_token_id]
             return cls + token_ids_a + sep + sep + token_ids_a + sep



       def 					lowercase__    (							self      :  Union[str, Any]		,    _UpperCAmelCase      :  List[int]		,    _UpperCAmelCase      :  Optional[List[int]] = None		,    _UpperCAmelCase      :  bool = False   )			->						List[int]:


             '''simple docstring'''




             if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=_UpperCAmelCase		,    token_ids_a=_UpperCAmelCase		,    already_has_special_tokens=_UpperCAmelCase   )

             if token_ids_a is None:
                   return [1] + ([0] * len(_UpperCAmelCase   )) + [1]
             return [1] + ([0] * len(_UpperCAmelCase   )) + [1, 1] + ([0] * len(_UpperCAmelCase   )) + [1]



       def 					lowercase__    (							self      :  Tuple		,    _UpperCAmelCase      :  List[int]		,    _UpperCAmelCase      :  Optional[List[int]] = None   )			->						List[int]:


             '''simple docstring'''




             UpperCAmelCase_      			=							[self.sep_token_id]
             UpperCAmelCase_      			=							[self.cls_token_id]

             if token_ids_a is None:
                   return len(cls + token_ids_a + sep   ) * [0]
             return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]



       @property
       def 					lowercase__    (							self      :  Dict   )			->						Any:


             '''simple docstring'''




             return len(self.fairseq_ids_to_tokens   )



       def 					lowercase__    (							self      :  int   )			->						Dict:


             '''simple docstring'''




             UpperCAmelCase_      			=							{self.convert_ids_to_tokens(_UpperCAmelCase   ): i for i in range(self.vocab_size   )}
             vocab.update(self.added_tokens_encoder   )
             return vocab



       def 					lowercase__    (							self      :  List[str]		,    _UpperCAmelCase      :  str   )			->						List[str]:


             '''simple docstring'''




             return self.sp_model.encode(_UpperCAmelCase		,    out_type=_UpperCAmelCase   )



       def 					lowercase__    (							self      :  str		,    _UpperCAmelCase      :  List[Any]   )			->						Any:


             '''simple docstring'''




             if token in self.fairseq_tokens_to_ids:
                   return self.fairseq_tokens_to_ids[token]
             else:
                   return self.unk_token_id



       def 					lowercase__    (							self      :  List[str]		,    _UpperCAmelCase      :  int   )			->						List[str]:


             '''simple docstring'''




             return self.fairseq_ids_to_tokens[index]



       def 					lowercase__    (							self      :  int		,    _UpperCAmelCase      :  int   )			->						Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_      			=							"".join(_UpperCAmelCase   ).replace(_UpperCAmelCase		,    " "   ).strip()
             return out_string







       def 					lowercase__    (							self      :  Optional[Any]		,    _UpperCAmelCase      :  str		,    _UpperCAmelCase      :  Optional[str] = None   )			->						Tuple[str]:


             '''simple docstring'''




             if not os.path.isdir(_UpperCAmelCase   ):
                   logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
                   return
             UpperCAmelCase_      			=							os.path.join(
                 _UpperCAmelCase		,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
             UpperCAmelCase_      			=							os.path.join(
                 _UpperCAmelCase		,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["monolingual_vocab_file"]		,    )

             if os.path.abspath(self.vocab_file   ) != os.path.abspath(_UpperCAmelCase   ) and os.path.isfile(self.vocab_file   ):
                   copyfile(self.vocab_file		,    _UpperCAmelCase   )
             elif not os.path.isfile(self.vocab_file   ):
                   with open(_UpperCAmelCase		,    "wb"   ) as fi:
                         UpperCAmelCase_      			=							self.sp_model.serialized_model_proto()
                         fi.write(_UpperCAmelCase   )

             if os.path.abspath(self.monolingual_vocab_file   ) != os.path.abspath(
                 _UpperCAmelCase   ) and os.path.isfile(self.monolingual_vocab_file   ):
                   copyfile(self.monolingual_vocab_file		,    _UpperCAmelCase   )
             elif not os.path.isfile(self.monolingual_vocab_file   ):
                   with open(_UpperCAmelCase		,    "w"		,    encoding="utf-8"   ) as fp:
                         for token in self.fairseq_tokens_to_ids:
                               if token not in self.all_special_tokens:
                                     fp.write(F"""{str(_UpperCAmelCase   )} \n"""   )

             return out_vocab_file, out_monolingual_vocab_file









"""simple docstring"""





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowerCamelCase       	=						datasets.utils.logging.get_logger(__name__)
@dataclass
class 				lowercase__       (   datasets.BuilderConfig				):







       '''simple docstring'''


       UpperCamelCase						=					None
       UpperCamelCase						=					"utf-8"
       UpperCamelCase						=					None
       UpperCamelCase						=					None
       UpperCamelCase						=					True  # deprecated
       UpperCamelCase						=					None  # deprecated
       UpperCamelCase						=					10 << 20  # 10MB
       UpperCamelCase						=					None



class 				lowercase__       (   datasets.ArrowBasedBuilder				):







       '''simple docstring'''


       UpperCamelCase						=					JsonConfig



       def 					lowercase__    (							self      :  List[Any]   )			->						Dict:


             '''simple docstring'''




             if self.config.block_size is not None:
                   logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"   )
                   UpperCAmelCase_      			=							self.config.block_size
             if self.config.use_threads is not True:
                   logger.warning(
                       "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."   )
             if self.config.newlines_in_values is not None:
                   raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"   )
             return datasets.DatasetInfo(features=self.config.features   )



       def 					lowercase__    (							self      :  Tuple		,    _UpperCAmelCase      :  Optional[int]   )			->						Optional[int]:


             '''simple docstring'''




             if not self.config.data_files:
                   raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""   )
             UpperCAmelCase_      			=							dl_manager.download_and_extract(self.config.data_files   )
             if isinstance(_UpperCAmelCase		,    (str, list, tuple)   ):
                   UpperCAmelCase_      			=							data_files
                   if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ):
                         UpperCAmelCase_      			=							[files]
                   UpperCAmelCase_      			=							[dl_manager.iter_files(_UpperCAmelCase   ) for file in files]
                   return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,    gen_kwargs={"files": files}   )]
             UpperCAmelCase_      			=							[]
             for split_name, files in data_files.items():
                   if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ):
                         UpperCAmelCase_      			=							[files]
                   UpperCAmelCase_      			=							[dl_manager.iter_files(_UpperCAmelCase   ) for file in files]
                   splits.append(datasets.SplitGenerator(name=_UpperCAmelCase		,    gen_kwargs={"files": files}   )   )
             return splits



       def 					lowercase__    (							self      :  str		,    _UpperCAmelCase      :  pa.Table   )			->						pa.Table:


             '''simple docstring'''




             if self.config.features is not None:
                   # adding missing columns
                   for column_name in set(self.config.features   ) - set(pa_table.column_names   ):
                         UpperCAmelCase_      			=							self.config.features.arrow_schema.field(_UpperCAmelCase   ).type
                         UpperCAmelCase_      			=							pa_table.append_column(_UpperCAmelCase		,    pa.array([None] * len(_UpperCAmelCase   )		,    type=_UpperCAmelCase   )   )
                   # more expensive cast to support nested structures with keys in a different order
                   # allows str <-> int/float or str to Audio for example
                   UpperCAmelCase_      			=							table_cast(_UpperCAmelCase		,    self.config.features.arrow_schema   )
             return pa_table







       def 					lowercase__    (							self      :  Optional[int]		,    _UpperCAmelCase      :  List[Any]   )			->						str:


             '''simple docstring'''




             for file_idx, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase   )   ):
                   # If the file is one json object and if we need to look at the list of items in one specific field
                   if self.config.field is not None:
                         with open(_UpperCAmelCase		,    encoding=self.config.encoding		,    errors=self.config.encoding_errors   ) as f:
                               UpperCAmelCase_      			=							json.load(_UpperCAmelCase   )

                         # We keep only the field we are interested in
                         UpperCAmelCase_      			=							dataset[self.config.field]

                         # We accept two format: a list of dicts or a dict of lists
                         if isinstance(_UpperCAmelCase		,    (list, tuple)   ):
                               UpperCAmelCase_      			=							set().union(*[row.keys() for row in dataset]   )
                               UpperCAmelCase_      			=							{col: [row.get(_UpperCAmelCase   ) for row in dataset] for col in keys}
                         else:
                               UpperCAmelCase_      			=							dataset
                         UpperCAmelCase_      			=							pa.Table.from_pydict(_UpperCAmelCase   )
                         yield file_idx, self._cast_table(_UpperCAmelCase   )

                   # If the file has one json object per line
                   else:
                         with open(_UpperCAmelCase		,    "rb"   ) as f:
                               UpperCAmelCase_      			=							0
                               # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                               # Set a default minimum value of 16kB if the chunk size is really small
                               UpperCAmelCase_      			=							max(self.config.chunksize // 32		,    16 << 10   )
                               UpperCAmelCase_      			=							(
                                   self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
                               )
                               while True:
                                     UpperCAmelCase_      			=							f.read(self.config.chunksize   )
                                     if not batch:
                                           break
                                     # Finish current line
                                     try:
                                           batch += f.readline()
                                     except (AttributeError, io.UnsupportedOperation):
                                           batch += readline(_UpperCAmelCase   )
                                     # PyArrow only accepts utf-8 encoded bytes
                                     if self.config.encoding != "utf-8":
                                           UpperCAmelCase_      			=							batch.decode(self.config.encoding		,    errors=_UpperCAmelCase   ).encode("utf-8"   )
                                     try:
                                           while True:
                                                 try:
                                                       UpperCAmelCase_      			=							paj.read_json(
                                                           io.BytesIO(_UpperCAmelCase   )		,    read_options=paj.ReadOptions(block_size=_UpperCAmelCase   )   )
                                                       break
                                                 except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                       if (
                                                           isinstance(_UpperCAmelCase		,    pa.ArrowInvalid   )
                                                           and "straddling" not in str(_UpperCAmelCase   )
                                                           or block_size > len(_UpperCAmelCase   )
                                                       ):
                                                             raise
                                                       else:
                                                             # Increase the block size in case it was too small.
                                                             # The block size will be reset for the next file.
                                                             logger.debug(
                                                                 F"""Batch of {len(_UpperCAmelCase   )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""   )
                                                             block_size *= 2
                                     except pa.ArrowInvalid as e:
                                           try:
                                                 with open(
                                                     _UpperCAmelCase		,    encoding=self.config.encoding		,    errors=self.config.encoding_errors   ) as f:
                                                       UpperCAmelCase_      			=							json.load(_UpperCAmelCase   )
                                           except json.JSONDecodeError:
                                                 logger.error(F"""Failed to read file '{file}' with error {type(_UpperCAmelCase   )}: {e}"""   )
                                                 raise e
                                           # If possible, parse the file as a list of json objects and exit the loop
                                           if isinstance(_UpperCAmelCase		,    _UpperCAmelCase   ):  # list is the only sequence type supported in JSON
                                                 try:
                                                       UpperCAmelCase_      			=							set().union(*[row.keys() for row in dataset]   )
                                                       UpperCAmelCase_      			=							{col: [row.get(_UpperCAmelCase   ) for row in dataset] for col in keys}
                                                       UpperCAmelCase_      			=							pa.Table.from_pydict(_UpperCAmelCase   )
                                                 except (pa.ArrowInvalid, AttributeError) as e:
                                                       logger.error(F"""Failed to read file '{file}' with error {type(_UpperCAmelCase   )}: {e}"""   )
                                                       raise ValueError(F"""Not able to read records in the JSON file at {file}."""   ) from None
                                                 yield file_idx, self._cast_table(_UpperCAmelCase   )
                                                 break
                                           else:
                                                 logger.error(F"""Failed to read file '{file}' with error {type(_UpperCAmelCase   )}: {e}"""   )
                                                 raise ValueError(
                                                     F"""Not able to read records in the JSON file at {file}. """
                                                     F"""You should probably indicate the field of the JSON file containing your records. """
                                                     F"""This JSON file contain the following fields: {str(list(dataset.keys()   )   )}. """
                                                     F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """   ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                     yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase   )
                                     batch_idx += 1




