





'''simple docstring'''

import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
				import PIL.Image

				from .features import FeatureType


UpperCAmelCase     		=   None
UpperCAmelCase     		=   "<" if sys.byteorder == "little" else ">"
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
UpperCAmelCase     		=   [
    np.dtype('|b1'),
    np.dtype('|u1'),
    np.dtype('<u2'),
    np.dtype('>u2'),
    np.dtype('<i2'),
    np.dtype('>i2'),
    np.dtype('<u4'),
    np.dtype('>u4'),
    np.dtype('<i4'),
    np.dtype('>i4'),
    np.dtype('<f4'),
    np.dtype('>f4'),
    np.dtype('<f8'),
    np.dtype('>f8'),
]

@dataclass
class __snake_case:


						'''simple docstring'''



						UpperCAmelCase							:   int									=		True
						UpperCAmelCase							:   Optional[int]									=		None
						# Automatically constructed
						UpperCAmelCase							:   Dict									=		"PIL.Image.Image"
						UpperCAmelCase							:   Union[str, Any]									=		pa.struct({"bytes": pa.binary(), "path": pa.string()} )
						UpperCAmelCase							:   Optional[int]									=		field(default="Image"		,							init=_SCREAMING_SNAKE_CASE		,							repr=_SCREAMING_SNAKE_CASE )


						def __call__( self				)     ->     Dict:
							return self.pa_type


						def     __snake_case   ( self     ,	A_				)     ->     dict:
							if config.PIL_AVAILABLE:
								import PIL.Image
							else:
								raise ImportError("""To support encoding images, please install 'Pillow'."""				)

							if isinstance(A_     ,	A_				):
								lowerCAmelCase   =     np.array(A_				)

							if isinstance(A_     ,	A_				):
								return {"path": value, "bytes": None}
							elif isinstance(A_     ,	A_				):
								return {"path": None, "bytes": value}
							elif isinstance(A_     ,	np.ndarray				):
								# convert the image array to PNG/TIFF bytes
								return encode_np_array(A_				)
							elif isinstance(A_     ,	PIL.Image.Image				):
								# convert the PIL image to bytes (default format is PNG/TIFF)
								return encode_pil_image(A_				)
							elif value.get("""path"""				) is not None and os.path.isfile(value["""path"""]				):
								# we set "bytes": None to not duplicate the data if they're already available locally
								return {"bytes": None, "path": value.get("""path"""				)}
							elif value.get("""bytes"""				) is not None or value.get("""path"""				) is not None:
								# store the image bytes, and path is used to infer the image format using the file extension
								return {"bytes": value.get("""bytes"""				), "path": value.get("""path"""				)}
							else:
								raise ValueError(
								    f'An image sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'				)


						def     __snake_case   ( self     ,	A_     ,	A_=None				)     ->     "PIL.Image.Image":
							if not self.decode:
								raise RuntimeError("""Decoding is disabled for this feature. Please use Image(decode=True) instead."""				)

							if config.PIL_AVAILABLE:
								import PIL.Image
							else:
								raise ImportError("""To support decoding images, please install 'Pillow'."""				)

							if token_per_repo_id is None:
								lowerCAmelCase   =     {}

							lowerCAmelCase   =     value["""path"""], value["""bytes"""]
							if bytes_ is None:
								if path is None:
									raise ValueError(f'An image should have one of \'path\' or \'bytes\' but both are None in {value}.'				)
								else:
									if is_local_path(A_				):
										lowerCAmelCase   =     PIL.Image.open(A_				)
									else:
										lowerCAmelCase   =     path.split("""::"""				)[-1]
										try:
											lowerCAmelCase   =     string_to_dict(A_     ,	config.HUB_DATASETS_URL				)["""repo_id"""]
											lowerCAmelCase   =     token_per_repo_id.get(A_				)
										except ValueError:
											lowerCAmelCase   =     None
										with xopen(A_     ,	"""rb"""     ,	use_auth_token=A_				) as f:
											lowerCAmelCase   =     BytesIO(f.read()				)
										lowerCAmelCase   =     PIL.Image.open(bytes_				)
							else:
								lowerCAmelCase   =     PIL.Image.open(BytesIO(bytes_				)				)
							image.load()  # to avoid "Too many open files" errors
							return image


						def     __snake_case   ( self				)     ->     Union["FeatureType", Dict[str, "FeatureType"]]:
							from .features import Value

							return (
							    self
							    if self.decode
							    else {
							        "bytes": Value("""binary"""				),
							        "path": Value("""string"""				),
							    }
							)


						def     __snake_case   ( self     ,	A_				)     ->     pa.StructArray:
							if pa.types.is_string(storage.type				):
								lowerCAmelCase   =     pa.array([None] * len(A_				)     ,	type=pa.binary()				)
								lowerCAmelCase   =     pa.StructArray.from_arrays([bytes_array, storage]     ,	["""bytes""", """path"""]     ,	mask=storage.is_null()				)
							elif pa.types.is_binary(storage.type				):
								lowerCAmelCase   =     pa.array([None] * len(A_				)     ,	type=pa.string()				)
								lowerCAmelCase   =     pa.StructArray.from_arrays([storage, path_array]     ,	["""bytes""", """path"""]     ,	mask=storage.is_null()				)
							elif pa.types.is_struct(storage.type				):
								if storage.type.get_field_index("""bytes"""				) >= 0:
									lowerCAmelCase   =     storage.field("""bytes"""				)
								else:
									lowerCAmelCase   =     pa.array([None] * len(A_				)     ,	type=pa.binary()				)
								if storage.type.get_field_index("""path"""				) >= 0:
									lowerCAmelCase   =     storage.field("""path"""				)
								else:
									lowerCAmelCase   =     pa.array([None] * len(A_				)     ,	type=pa.string()				)
								lowerCAmelCase   =     pa.StructArray.from_arrays([bytes_array, path_array]     ,	["""bytes""", """path"""]     ,	mask=storage.is_null()				)
							elif pa.types.is_list(storage.type				):
								lowerCAmelCase   =     pa.array(
								    [encode_np_array(np.array(A_				)				)["""bytes"""] if arr is not None else None for arr in storage.to_pylist()]     ,	type=pa.binary()     ,	)
								lowerCAmelCase   =     pa.array([None] * len(A_				)     ,	type=pa.string()				)
								lowerCAmelCase   =     pa.StructArray.from_arrays(
								    [bytes_array, path_array]     ,	["""bytes""", """path"""]     ,	mask=bytes_array.is_null()				)
							return array_cast(A_     ,	self.pa_type				)




						def     __snake_case   ( self     ,	A_				)     ->     pa.StructArray:

							@no_op_if_value_is_null
							def path_to_bytes(A_				):
								with xopen(A_     ,	"""rb"""				) as f:
									lowerCAmelCase   =     f.read()
								return bytes_

							lowerCAmelCase   =     pa.array(
							    [
							        (path_to_bytes(x["""path"""]				) if x["""bytes"""] is None else x["""bytes"""]) if x is not None else None
							        for x in storage.to_pylist()
							    ]     ,	type=pa.binary()     ,	)
							lowerCAmelCase   =     pa.array(
							    [os.path.basename(A_				) if path is not None else None for path in storage.field("""path"""				).to_pylist()]     ,	type=pa.string()     ,	)
							lowerCAmelCase   =     pa.StructArray.from_arrays([bytes_array, path_array]     ,	["""bytes""", """path"""]     ,	mask=bytes_array.is_null()				)
							return array_cast(A_     ,	self.pa_type				)







def 		_snake_case		( )      ->						str:







	"""simple docstring"""





	if config.PIL_AVAILABLE:
		import PIL.Image
	else:
		raise ImportError("""To support encoding images, please install 'Pillow'."""							)

	global _IMAGE_COMPRESSION_FORMATS
	if _IMAGE_COMPRESSION_FORMATS is None:
		PIL.Image.init()
		lowerCAmelCase   =     list(set(PIL.Image.OPEN.keys()							) & set(PIL.Image.SAVE.keys()							)							)
	return _IMAGE_COMPRESSION_FORMATS






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						"PIL.Image.Image"							)      ->						Tuple:







	"""simple docstring"""
	lowerCAmelCase   =     BytesIO()
	if image.format in list_image_compression_formats():
		lowerCAmelCase   =     image.format
	else:
		lowerCAmelCase   =     """PNG""" if image.mode in ["""1""", """L""", """LA""", """RGB""", """RGBA"""] else """TIFF"""
	image.save(A__							,		format=A__							)
	return buffer.getvalue()






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						"PIL.Image.Image"							)      ->						List[Any]:







	"""simple docstring"""





	if hasattr(A__							,		"""filename"""							) and image.filename != "":
		return {"path": image.filename, "bytes": None}
	else:
		return {"path": None, "bytes": image_to_bytes(A__							)}






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						np.ndarray							)      ->						List[Any]:







	"""simple docstring"""





	if config.PIL_AVAILABLE:
		import PIL.Image
	else:
		raise ImportError("""To support encoding images, please install 'Pillow'."""							)

	lowerCAmelCase   =     array.dtype
	lowerCAmelCase   =     dtype.byteorder if dtype.byteorder != """=""" else _NATIVE_BYTEORDER
	lowerCAmelCase   =     dtype.kind
	lowerCAmelCase   =     dtype.itemsize

	lowerCAmelCase   =     None

	# Multi-channel array case (only np.dtype("|u1") is allowed)
	if array.shape[2:]:
		lowerCAmelCase   =     np.dtype("""|u1"""							)
		if dtype_kind not in ["u", "i"]:
			raise TypeError(
			    f'Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays.'							)
		if dtype is not dest_dtype:
			warnings.warn(f'Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\''							)
		# Exact match
	elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
		lowerCAmelCase   =     dtype
	else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
		while dtype_itemsize >= 1:
			lowerCAmelCase   =     dtype_byteorder + dtype_kind + str(A__							)
			lowerCAmelCase   =     np.dtype(A__							)
			if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
				warnings.warn(f'Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\''							)
				break
			else:
				dtype_itemsize //= 2

	if dest_dtype is None:
		raise TypeError(
		    f'Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}'							)
	lowerCAmelCase   =     PIL.Image.fromarray(array.astype(A__							)							)
	return {"path": None, "bytes": image_to_bytes(A__							)}






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						Union[List[str], List[dict], List[np.ndarray], List["PIL.Image.Image"]]							)      ->						int:







	"""simple docstring"""





	if config.PIL_AVAILABLE:
		import PIL.Image
	else:
		raise ImportError("""To support encoding images, please install 'Pillow'."""							)

	if objs:
		lowerCAmelCase   =     first_non_null_value(A__							)
		if isinstance(A__							,		A__							):
			return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
		if isinstance(A__							,		np.ndarray							):
			lowerCAmelCase   =     no_op_if_value_is_null(A__							)
			return [obj_to_image_dict_func(A__							) for obj in objs]
		elif isinstance(A__							,		PIL.Image.Image							):
			lowerCAmelCase   =     no_op_if_value_is_null(A__							)
			return [obj_to_image_dict_func(A__							) for obj in objs]
		else:
			return objs
	else:
		return objs




'''simple docstring'''

import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase     		=   logging.get_logger(__name__)

UpperCAmelCase     		=   '▁'

UpperCAmelCase     		=   {
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

UpperCAmelCase     		=   {
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

UpperCAmelCase     		=   {
    'facebook/m2m100_418M': 1024,
}

# fmt: off
UpperCAmelCase     		=   {
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}

class __snake_case(				_lowerCAmelCase ):


						'''simple docstring'''



						UpperCAmelCase							:   Optional[int]									=		VOCAB_FILES_NAMES
						UpperCAmelCase							:   Optional[Any]									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase							:   Optional[int]									=		PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase							:   int									=		["input_ids", "attention_mask"]

						UpperCAmelCase							:   List[int]									=		[]
						UpperCAmelCase							:   List[int]									=		[]


						def __init__( self     ,	A_     ,	A_     ,	A_=None     ,	A_=None     ,	A_="<s>"     ,	A_="</s>"     ,	A_="</s>"     ,	A_="<pad>"     ,	A_="<unk>"     ,	A_="m2m100"     ,	A_ = None     ,	A_=8     ,	**A_     ,	)     ->     None:
							lowerCAmelCase   =     {} if sp_model_kwargs is None else sp_model_kwargs

							lowerCAmelCase   =     language_codes
							lowerCAmelCase   =     FAIRSEQ_LANGUAGE_CODES[language_codes]
							lowerCAmelCase   =     {lang_code: f'__{lang_code}__' for lang_code in fairseq_language_code}

							lowerCAmelCase   =     kwargs.get("""additional_special_tokens"""     ,	[]				)
							kwargs["additional_special_tokens"] += [
							    self.get_lang_token(A_				)
							    for lang_code in fairseq_language_code
							    if self.get_lang_token(A_				) not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    src_lang=A_     ,	tgt_lang=A_     ,	bos_token=A_     ,	eos_token=A_     ,	sep_token=A_     ,	unk_token=A_     ,	pad_token=A_     ,	language_codes=A_     ,	sp_model_kwargs=self.sp_model_kwargs     ,	num_madeup_words=A_     ,	**A_     ,	)

							lowerCAmelCase   =     vocab_file
							lowerCAmelCase   =     load_json(A_				)
							lowerCAmelCase   =     {v: k for k, v in self.encoder.items()}
							lowerCAmelCase   =     spm_file
							lowerCAmelCase   =     load_spm(A_     ,	self.sp_model_kwargs				)

							lowerCAmelCase   =     len(self.encoder				)

							lowerCAmelCase   =     {
							    self.get_lang_token(A_				): self.encoder_size + i for i, lang_code in enumerate(A_				)
							}
							lowerCAmelCase   =     {lang_code: self.encoder_size + i for i, lang_code in enumerate(A_				)}
							lowerCAmelCase   =     {v: k for k, v in self.lang_token_to_id.items()}

							lowerCAmelCase   =     src_lang if src_lang is not None else """en"""
							lowerCAmelCase   =     tgt_lang
							lowerCAmelCase   =     self.get_lang_id(self._src_lang				)
							self.set_src_lang_special_tokens(self._src_lang				)

							lowerCAmelCase   =     num_madeup_words


						@property
						def     __snake_case   ( self				)     ->     int:
							return len(self.encoder				) + len(self.lang_token_to_id				)


						@property
						def     __snake_case   ( self				)     ->     str:
							return self._src_lang


						@src_lang.setter
						def     __snake_case   ( self     ,	A_				)     ->     None:
							lowerCAmelCase   =     new_src_lang
							self.set_src_lang_special_tokens(self._src_lang				)


						def     __snake_case   ( self     ,	A_				)     ->     List[str]:
							return self.sp_model.encode(A_     ,	out_type=A_				)


						def     __snake_case   ( self     ,	A_				)     ->     Any:
							if token in self.lang_token_to_id:
								return self.lang_token_to_id[token]
							return self.encoder.get(A_     ,	self.encoder[self.unk_token]				)


						def     __snake_case   ( self     ,	A_				)     ->     str:
							if index in self.id_to_lang_token:
								return self.id_to_lang_token[index]
							return self.decoder.get(A_     ,	self.unk_token				)


						def     __snake_case   ( self     ,	A_				)     ->     List[str]:
							lowerCAmelCase   =     []
							lowerCAmelCase   =     """"""
							for token in tokens:
								# make sure that special tokens are not decoded using sentencepiece model
								if token in self.all_special_tokens:
									out_string += self.sp_model.decode(A_				) + token
									lowerCAmelCase   =     []
								else:
									current_sub_tokens.append(A_				)
							out_string += self.sp_model.decode(A_				)
							return out_string.strip()


						def     __snake_case   ( self     ,	A_     ,	A_ = None     ,	A_ = False				)     ->     List[int]:

							if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=A_     ,	token_ids_a=A_     ,	already_has_special_tokens=A_				)

							lowerCAmelCase   =     [1] * len(self.prefix_tokens				)
							lowerCAmelCase   =     [1] * len(self.suffix_tokens				)
							if token_ids_a is None:
								return prefix_ones + ([0] * len(A_				)) + suffix_ones
							return prefix_ones + ([0] * len(A_				)) + ([0] * len(A_				)) + suffix_ones


						def     __snake_case   ( self     ,	A_     ,	A_ = None				)     ->     List[int]:
							if token_ids_a is None:
								return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


						def     __snake_case   ( self				)     ->     Dict:
							lowerCAmelCase   =     {self.convert_ids_to_tokens(A_				): i for i in range(self.vocab_size				)}
							vocab.update(self.added_tokens_encoder				)
							return vocab


						def __getstate__( self				)     ->     Dict:
							lowerCAmelCase   =     self.__dict__.copy()
							lowerCAmelCase   =     None
							return state


						def __setstate__( self     ,	A_				)     ->     None:
							lowerCAmelCase   =     d

							# for backward compatibility
							if not hasattr(self     ,	"""sp_model_kwargs"""				):
								lowerCAmelCase   =     {}

							lowerCAmelCase   =     load_spm(self.spm_file     ,	self.sp_model_kwargs				)


						def     __snake_case   ( self     ,	A_     ,	A_ = None				)     ->     Tuple[str]:
							lowerCAmelCase   =     Path(A_				)
							if not save_dir.is_dir():
								raise OSError(f'{save_directory} should be a directory'				)
							lowerCAmelCase   =     save_dir / (
							    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
							)
							lowerCAmelCase   =     save_dir / (
							    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
							)

							save_json(self.encoder     ,	A_				)

							if os.path.abspath(self.spm_file				) != os.path.abspath(A_				) and os.path.isfile(self.spm_file				):
								copyfile(self.spm_file     ,	A_				)
							elif not os.path.isfile(self.spm_file				):
								with open(A_     ,	"""wb"""				) as fi:
									lowerCAmelCase   =     self.sp_model.serialized_model_proto()
									fi.write(A_				)

							return (str(A_				), str(A_				))


						def     __snake_case   ( self     ,	A_     ,	A_ = "en"     ,	A_ = None     ,	A_ = "ro"     ,	**A_     ,	)     ->     BatchEncoding:
							lowerCAmelCase   =     src_lang
							lowerCAmelCase   =     tgt_lang
							self.set_src_lang_special_tokens(self.src_lang				)
							return super().prepare_seqaseq_batch(A_     ,	A_     ,	**A_				)


						def     __snake_case   ( self     ,	A_     ,	A_     ,	A_     ,	**A_				)     ->     str:
							if src_lang is None or tgt_lang is None:
								raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""				)
							lowerCAmelCase   =     src_lang
							lowerCAmelCase   =     self(A_     ,	add_special_tokens=A_     ,	**A_				)
							lowerCAmelCase   =     self.get_lang_id(A_				)
							lowerCAmelCase   =     tgt_lang_id
							return inputs


						def     __snake_case   ( self				)     ->     Any:
							self.set_src_lang_special_tokens(self.src_lang				)


						def     __snake_case   ( self				)     ->     Optional[int]:
							self.set_tgt_lang_special_tokens(self.tgt_lang				)


						def     __snake_case   ( self     ,	A_				)     ->     None:
							lowerCAmelCase   =     self.get_lang_token(A_				)
							lowerCAmelCase   =     self.lang_token_to_id[lang_token]
							lowerCAmelCase   =     [self.cur_lang_id]
							lowerCAmelCase   =     [self.eos_token_id]


						def     __snake_case   ( self     ,	A_				)     ->     None:
							lowerCAmelCase   =     self.get_lang_token(A_				)
							lowerCAmelCase   =     self.lang_token_to_id[lang_token]
							lowerCAmelCase   =     [self.cur_lang_id]
							lowerCAmelCase   =     [self.eos_token_id]


						def     __snake_case   ( self     ,	A_				)     ->     str:
							return self.lang_code_to_token[lang]




						def     __snake_case   ( self     ,	A_				)     ->     int:
							lowerCAmelCase   =     self.get_lang_token(A_				)
							return self.lang_token_to_id[lang_token]







def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						str							,		_SCREAMING_SNAKE_CASE					:						Dict[str, Any]							)      ->						sentencepiece.SentencePieceProcessor:







	"""simple docstring"""
	lowerCAmelCase   =     sentencepiece.SentencePieceProcessor(**_SCREAMING_SNAKE_CASE							)
	spm.Load(str(_SCREAMING_SNAKE_CASE							)							)
	return spm






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						str							)      ->						Union[Dict, List]:







	"""simple docstring"""
	with open(_SCREAMING_SNAKE_CASE							,		"""r"""							) as f:
		return json.load(_SCREAMING_SNAKE_CASE							)






def 		_snake_case		( _SCREAMING_SNAKE_CASE					:						Dict							,		_SCREAMING_SNAKE_CASE					:						str							)      ->						None:







	"""simple docstring"""
	with open(_SCREAMING_SNAKE_CASE							,		"""w"""							) as f:
		json.dump(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							,		indent=2							)