






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


A__	:    Any			 = logging.get_logger(__name__)

A__	:    int			 = '''▁'''

A__	:    str			 = {'''vocab_file''': '''sentencepiece.bpe.model'''}

A__	:    Dict			 = {
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

A__	:    Dict			 = {
    '''facebook/mbart-large-en-ro''': 1_024,
    '''facebook/mbart-large-cc25''': 1_024,
}

# fmt: off
A__	:    Optional[int]			 = ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']







class        lowercase__      (							_UpperCAmelCase       ):
			_UpperCAmelCase		:List[Any]             =			VOCAB_FILES_NAMES
			_UpperCAmelCase		:Dict             =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase		:List[str]             =			PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase		:Any             =			['input_ids', 'attention_mask']

			_UpperCAmelCase		:List[int]             =			[]
			_UpperCAmelCase		:List[int]             =			[]


			def __init__(				self					:  Dict       ,	snake_case__					:  Tuple       ,	snake_case__					:  Optional[Any]="<s>"       ,	snake_case__					:  List[Any]="</s>"       ,	snake_case__					:  Optional[Any]="</s>"       ,	snake_case__					:  str="<s>"       ,	snake_case__					:  Union[str, Any]="<unk>"       ,	snake_case__					:  List[str]="<pad>"       ,	snake_case__					:  List[str]="<mask>"       ,	snake_case__					:  Union[str, Any]=None       ,	snake_case__					:  Dict=None       ,	snake_case__					:  Tuple=None       ,	snake_case__					:  Tuple = None       ,	snake_case__					:  Optional[Any]=None       ,	**snake_case__					:  Any       ,	):
										# Mask token behave like a normal word, i.e. include the space before it
										lowerCamelCase_	:							int									=AddedToken(SCREAMING_SNAKE_CASE_       ,	lstrip=SCREAMING_SNAKE_CASE_       ,	rstrip=SCREAMING_SNAKE_CASE_ ) if isinstance(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_ ) else mask_token

										lowerCamelCase_	:							Optional[Any]									={} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    bos_token=SCREAMING_SNAKE_CASE_       ,	eos_token=SCREAMING_SNAKE_CASE_       ,	unk_token=SCREAMING_SNAKE_CASE_       ,	sep_token=SCREAMING_SNAKE_CASE_       ,	cls_token=SCREAMING_SNAKE_CASE_       ,	pad_token=SCREAMING_SNAKE_CASE_       ,	mask_token=SCREAMING_SNAKE_CASE_       ,	tokenizer_file=SCREAMING_SNAKE_CASE_       ,	src_lang=SCREAMING_SNAKE_CASE_       ,	tgt_lang=SCREAMING_SNAKE_CASE_       ,	additional_special_tokens=SCREAMING_SNAKE_CASE_       ,	sp_model_kwargs=self.sp_model_kwargs       ,	**SCREAMING_SNAKE_CASE_       ,	)

										lowerCamelCase_	:							List[str]									=spm.SentencePieceProcessor(**self.sp_model_kwargs )
										self.sp_model.Load(str(SCREAMING_SNAKE_CASE_ ) )
										lowerCamelCase_	:							str									=vocab_file

										# Original fairseq vocab and spm vocab must be "aligned":
										# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
										# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
										# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
										# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

										# Mimic fairseq token-to-id alignment for the first 4 token
										lowerCamelCase_	:							int									={"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

										# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
										lowerCamelCase_	:							str									=1

										lowerCamelCase_	:							List[str]									=len(self.sp_model )
										lowerCamelCase_	:							List[str]									={
										    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(SCREAMING_SNAKE_CASE_ )
										}
										lowerCamelCase_	:							Dict									={v: k for k, v in self.lang_code_to_id.items()}
										lowerCamelCase_	:							Union[str, Any]									=len(self.sp_model ) + len(self.lang_code_to_id ) + self.fairseq_offset

										self.fairseq_tokens_to_ids.update(self.lang_code_to_id )
										lowerCamelCase_	:							int									={v: k for k, v in self.fairseq_tokens_to_ids.items()}
										lowerCamelCase_	:							Tuple									=list(self.lang_code_to_id.keys() )

										if additional_special_tokens is not None:
																	# Only add those special tokens if they are not already there.
																	self._additional_special_tokens.extend(
																	    [t for t in additional_special_tokens if t not in self._additional_special_tokens] )

										lowerCamelCase_	:							List[Any]									=src_lang if src_lang is not None else """en_XX"""
										lowerCamelCase_	:							str									=self.lang_code_to_id[self._src_lang]
										lowerCamelCase_	:							Union[str, Any]									=tgt_lang
										self.set_src_lang_special_tokens(self._src_lang )


			def __getstate__(				self					:  List[str] ):
										lowerCamelCase_	:							str									=self.__dict__.copy()
										lowerCamelCase_	:							List[Any]									=None
										lowerCamelCase_	:							int									=self.sp_model.serialized_model_proto()
										return state


			def __setstate__(				self					:  List[str]       ,	snake_case__					:  List[str] ):
										lowerCamelCase_	:							Union[str, Any]									=d

										# for backward compatibility
										if not hasattr(self       ,	"sp_model_kwargs" ):
																	lowerCamelCase_	:							Any									={}

										lowerCamelCase_	:							List[Any]									=spm.SentencePieceProcessor(**self.sp_model_kwargs )
										self.sp_model.LoadFromSerializedProto(self.sp_model_proto )


			@property
			def      UpperCAmelCase__						(				self					:  Optional[Any] ):
										return len(self.sp_model ) + len(self.lang_code_to_id ) + self.fairseq_offset + 1  # Plus 1 for the mask token


			@property
			def      UpperCAmelCase__						(				self					:  List[str] ):
										return self._src_lang


			@src_lang.setter
			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  Optional[int] ):
										lowerCamelCase_	:							Dict									=new_src_lang
										self.set_src_lang_special_tokens(self._src_lang )


			def      UpperCAmelCase__						(				self					:  List[Any]       ,	snake_case__					:  Optional[Any]       ,	snake_case__					:  Optional[Any] = None       ,	snake_case__					:  Union[str, Any] = False ):

										if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=SCREAMING_SNAKE_CASE_       ,	token_ids_a=SCREAMING_SNAKE_CASE_       ,	already_has_special_tokens=SCREAMING_SNAKE_CASE_ )

										lowerCamelCase_	:							Union[str, Any]									=[1] * len(self.prefix_tokens )
										lowerCamelCase_	:							Union[str, Any]									=[1] * len(self.suffix_tokens )
										if token_ids_a is None:
																	return prefix_ones + ([0] * len(SCREAMING_SNAKE_CASE_ )) + suffix_ones
										return prefix_ones + ([0] * len(SCREAMING_SNAKE_CASE_ )) + ([0] * len(SCREAMING_SNAKE_CASE_ )) + suffix_ones


			def      UpperCAmelCase__						(				self					:  str       ,	snake_case__					:  List[str]       ,	snake_case__					:  str = None ):
										if token_ids_a is None:
																	return self.prefix_tokens + token_ids_a + self.suffix_tokens
										# We don't expect to process pairs, but leave the pair logic for API consistency
										return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


			def      UpperCAmelCase__						(				self					:  List[str]       ,	snake_case__					:  List[str]       ,	snake_case__					:  List[Any] = None ):
										lowerCamelCase_	:							Optional[int]									=[self.sep_token_id]
										lowerCamelCase_	:							Tuple									=[self.cls_token_id]

										if token_ids_a is None:
																	return len(cls + token_ids_a + sep ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]


			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  List[Any]       ,	snake_case__					:  str       ,	snake_case__					:  List[str]       ,	snake_case__					:  Union[str, Any]       ,	**snake_case__					:  int ):
										if src_lang is None or tgt_lang is None:
																	raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model" )
										lowerCamelCase_	:							Any									=src_lang
										lowerCamelCase_	:							Union[str, Any]									=self(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_       ,	return_tensors=SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_ )
										lowerCamelCase_	:							Optional[int]									=self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ )
										lowerCamelCase_	:							Union[str, Any]									=tgt_lang_id
										return inputs


			def      UpperCAmelCase__						(				self					:  Optional[int] ):
										lowerCamelCase_	:							Union[str, Any]									={self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_ ): i for i in range(self.vocab_size )}
										vocab.update(self.added_tokens_encoder )
										return vocab


			def      UpperCAmelCase__						(				self					:  Dict       ,	snake_case__					:  str ):
										return self.sp_model.encode(SCREAMING_SNAKE_CASE_       ,	out_type=SCREAMING_SNAKE_CASE_ )


			def      UpperCAmelCase__						(				self					:  Optional[int]       ,	snake_case__					:  List[Any] ):
										if token in self.fairseq_tokens_to_ids:
																	return self.fairseq_tokens_to_ids[token]
										lowerCamelCase_	:							List[str]									=self.sp_model.PieceToId(SCREAMING_SNAKE_CASE_ )

										# Need to return unknown token if the SP model returned 0
										return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


			def      UpperCAmelCase__						(				self					:  Optional[Any]       ,	snake_case__					:  List[str] ):
										if index in self.fairseq_ids_to_tokens:
																	return self.fairseq_ids_to_tokens[index]
										return self.sp_model.IdToPiece(index - self.fairseq_offset )


			def      UpperCAmelCase__						(				self					:  Union[str, Any]       ,	snake_case__					:  Tuple ):
										lowerCamelCase_	:							List[Any]									="""""".join(SCREAMING_SNAKE_CASE_ ).replace(SCREAMING_SNAKE_CASE_       ,	" " ).strip()
										return out_string


			def      UpperCAmelCase__						(				self					:  List[Any]       ,	snake_case__					:  List[Any]       ,	snake_case__					:  Union[str, Any] = None ):
										if not os.path.isdir(SCREAMING_SNAKE_CASE_ ):
																	logger.error(F"""Vocabulary path ({save_directory}) should be a directory""" )
																	return
										lowerCamelCase_	:							List[str]									=os.path.join(
										    SCREAMING_SNAKE_CASE_       ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

										if os.path.abspath(self.vocab_file ) != os.path.abspath(SCREAMING_SNAKE_CASE_ ) and os.path.isfile(self.vocab_file ):
																	copyfile(self.vocab_file       ,	SCREAMING_SNAKE_CASE_ )
										elif not os.path.isfile(self.vocab_file ):
																	with open(SCREAMING_SNAKE_CASE_       ,	"wb" ) as fi:
																								lowerCamelCase_	:							Dict									=self.sp_model.serialized_model_proto()
																								fi.write(SCREAMING_SNAKE_CASE_ )

										return (out_vocab_file,)


			def      UpperCAmelCase__						(				self					:  List[Any]       ,	snake_case__					:  Dict       ,	snake_case__					:  List[str] = "en_XX"       ,	snake_case__					:  Dict = None       ,	snake_case__					:  List[Any] = "ro_RO"       ,	**snake_case__					:  Optional[int]       ,	):
										lowerCamelCase_	:							str									=src_lang
										lowerCamelCase_	:							Optional[int]									=tgt_lang
										return super().prepare_seqaseq_batch(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_ )


			def      UpperCAmelCase__						(				self					:  Dict ):
										return self.set_src_lang_special_tokens(self.src_lang )


			def      UpperCAmelCase__						(				self					:  Dict ):
										return self.set_tgt_lang_special_tokens(self.tgt_lang )


			def      UpperCAmelCase__						(				self					:  Union[str, Any]       ,	snake_case__					:  List[Any] ):
										lowerCamelCase_	:							Any									=self.lang_code_to_id[src_lang]
										lowerCamelCase_	:							Dict									=[]
										lowerCamelCase_	:							List[Any]									=[self.eos_token_id, self.cur_lang_code]


			def      UpperCAmelCase__						(				self					:  List[str]       ,	snake_case__					:  List[str] ):
										lowerCamelCase_	:							str									=self.lang_code_to_id[lang]
										lowerCamelCase_	:							Union[str, Any]									=[]
										lowerCamelCase_	:							List[str]									=[self.eos_token_id, self.cur_lang_code]











"""simple docstring"""



import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows

def 			_snake_case    (					lowerCamelCase__     :   Any  )    ->     Union[str, Any]:  # picklable for multiprocessing
							return i + 1

@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def 			_snake_case    (					)    ->     List[Any]:
							with parallel_backend("spark"  ):
														assert ParallelBackendConfig.backend_name == "spark"

							lowerCamelCase_	:							Optional[Any]									=[1, 2, 3]
							with pytest.raises(lowerCamelCase__  ):
														with parallel_backend("unsupported backend"  ):
																					map_nested(lowerCamelCase__    ,						lowerCamelCase__    ,						num_proc=2  )

							with pytest.raises(lowerCamelCase__  ):
														with parallel_backend("unsupported backend"  ):
																					map_nested(lowerCamelCase__    ,						lowerCamelCase__    ,						num_proc=-1  )

@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize("num_proc"    ,						[2, -1]  )
def 			_snake_case    (					lowerCamelCase__     :   Tuple  )    ->     Optional[Any]:
							lowerCamelCase_	:							str									=[1, 2]
							lowerCamelCase_	:							List[str]									={"a": 1, "b": 2}
							lowerCamelCase_	:							List[str]									={"a": [1, 2], "b": [3, 4]}
							lowerCamelCase_	:							Optional[int]									={"a": {"1": 1}, "b": 2}
							lowerCamelCase_	:							int									={"a": 1, "b": 2, "c": 3, "d": 4}
							lowerCamelCase_	:							Optional[int]									=[2, 3]
							lowerCamelCase_	:							List[Any]									={"a": 2, "b": 3}
							lowerCamelCase_	:							int									={"a": [2, 3], "b": [4, 5]}
							lowerCamelCase_	:							str									={"a": {"1": 2}, "b": 3}
							lowerCamelCase_	:							Dict									={"a": 2, "b": 3, "c": 4, "d": 5}

							with parallel_backend("spark"  ):
														assert map_nested(lowerCamelCase__    ,						lowerCamelCase__    ,						num_proc=lowerCamelCase__  ) == expected_map_nested_sa
														assert map_nested(lowerCamelCase__    ,						lowerCamelCase__    ,						num_proc=lowerCamelCase__  ) == expected_map_nested_sa
														assert map_nested(lowerCamelCase__    ,						lowerCamelCase__    ,						num_proc=lowerCamelCase__  ) == expected_map_nested_sa
														assert map_nested(lowerCamelCase__    ,						lowerCamelCase__    ,						num_proc=lowerCamelCase__  ) == expected_map_nested_sa
														assert map_nested(lowerCamelCase__    ,						lowerCamelCase__    ,						num_proc=lowerCamelCase__  ) == expected_map_nested_sa



