


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__  :Any             =							logging.get_logger(__name__)
lowerCAmelCase__  :List[Any]             =							{'''vocab_file''': '''spiece.model'''}

lowerCAmelCase__  :Tuple             =							{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

lowerCAmelCase__  :Union[str, Any]             =							{
    '''albert-base-v1''': 5_1_2,
    '''albert-large-v1''': 5_1_2,
    '''albert-xlarge-v1''': 5_1_2,
    '''albert-xxlarge-v1''': 5_1_2,
    '''albert-base-v2''': 5_1_2,
    '''albert-large-v2''': 5_1_2,
    '''albert-xlarge-v2''': 5_1_2,
    '''albert-xxlarge-v2''': 5_1_2,
}

lowerCAmelCase__  :List[Any]             =							'''▁'''

class 		__a				(    UpperCAmelCase      ):
		_a     :  List[str]           =     VOCAB_FILES_NAMES
		_a     :  int           =     PRETRAINED_VOCAB_FILES_MAP
		_a     :  Optional[int]           =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		def __init__(      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE=True      ,   _SCREAMING_SNAKE_CASE=True      ,   _SCREAMING_SNAKE_CASE=False      ,   _SCREAMING_SNAKE_CASE="[CLS]"      ,   _SCREAMING_SNAKE_CASE="[SEP]"      ,   _SCREAMING_SNAKE_CASE="<unk>"      ,   _SCREAMING_SNAKE_CASE="[SEP]"      ,   _SCREAMING_SNAKE_CASE="<pad>"      ,   _SCREAMING_SNAKE_CASE="[CLS]"      ,   _SCREAMING_SNAKE_CASE="[MASK]"      ,   _SCREAMING_SNAKE_CASE = None      ,   **_SCREAMING_SNAKE_CASE      ,   )			->	None:




									"""simple docstring"""





									_UpperCAmelCase					=      (
									    AddedToken(_SCREAMING_SNAKE_CASE      ,   lstrip=_SCREAMING_SNAKE_CASE      ,   rstrip=_SCREAMING_SNAKE_CASE      ,   normalized=_SCREAMING_SNAKE_CASE						)
									    if isinstance(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)
									    else mask_token
									)

									_UpperCAmelCase					=      {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=_SCREAMING_SNAKE_CASE      ,   remove_space=_SCREAMING_SNAKE_CASE      ,   keep_accents=_SCREAMING_SNAKE_CASE      ,   bos_token=_SCREAMING_SNAKE_CASE      ,   eos_token=_SCREAMING_SNAKE_CASE      ,   unk_token=_SCREAMING_SNAKE_CASE      ,   sep_token=_SCREAMING_SNAKE_CASE      ,   pad_token=_SCREAMING_SNAKE_CASE      ,   cls_token=_SCREAMING_SNAKE_CASE      ,   mask_token=_SCREAMING_SNAKE_CASE      ,   sp_model_kwargs=self.sp_model_kwargs      ,   **_SCREAMING_SNAKE_CASE      ,   )

									_UpperCAmelCase					=      do_lower_case
									_UpperCAmelCase					=      remove_space
									_UpperCAmelCase					=      keep_accents
									_UpperCAmelCase					=      vocab_file

									_UpperCAmelCase					=      spm.SentencePieceProcessor(**self.sp_model_kwargs						)
									self.sp_model.Load(_SCREAMING_SNAKE_CASE						)
		@property
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									return len(self.sp_model						)
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE						): i for i in range(self.vocab_size						)}
									vocab.update(self.added_tokens_encoder						)
									return vocab
		def __getstate__(      self						)			->	int:




									"""simple docstring"""





									_UpperCAmelCase					=      self.__dict__.copy()
									_UpperCAmelCase					=      None
									return state
		def __setstate__(      self      ,   _SCREAMING_SNAKE_CASE						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      d

									# for backward compatibility
									if not hasattr(self      ,   'sp_model_kwargs'						):
																_UpperCAmelCase					=      {}

									_UpperCAmelCase					=      spm.SentencePieceProcessor(**self.sp_model_kwargs						)
									self.sp_model.Load(self.vocab_file						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Optional[Any]:




									"""simple docstring"""





									if self.remove_space:
																_UpperCAmelCase					=      ' '.join(inputs.strip().split()						)
									else:
																_UpperCAmelCase					=      inputs
									_UpperCAmelCase					=      outputs.replace('``'      ,   '"'						).replace('\'\''      ,   '"'						)

									if not self.keep_accents:
																_UpperCAmelCase					=      unicodedata.normalize('NFKD'      ,   _SCREAMING_SNAKE_CASE						)
																_UpperCAmelCase					=      ''.join([c for c in outputs if not unicodedata.combining(_SCREAMING_SNAKE_CASE						)]						)
									if self.do_lower_case:
																_UpperCAmelCase					=      outputs.lower()

									return outputs
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	List[str]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.preprocess_text(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      self.sp_model.encode(_SCREAMING_SNAKE_CASE      ,   out_type=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      []
									for piece in pieces:
																if len(_SCREAMING_SNAKE_CASE						) > 1 and piece[-1] == str(','						) and piece[-2].isdigit():
																							_UpperCAmelCase					=      self.sp_model.EncodeAsPieces(piece[:-1].replace(_SCREAMING_SNAKE_CASE      ,   ''						)						)
																							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																														if len(cur_pieces[0]						) == 1:
																																					_UpperCAmelCase					=      cur_pieces[1:]
																														else:
																																					_UpperCAmelCase					=      cur_pieces[0][1:]
																							cur_pieces.append(piece[-1]						)
																							new_pieces.extend(_SCREAMING_SNAKE_CASE						)
																else:
																							new_pieces.append(_SCREAMING_SNAKE_CASE						)

									return new_pieces
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	int:




									"""simple docstring"""





									return self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Tuple:




									"""simple docstring"""





									return self.sp_model.IdToPiece(_SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	int:




									"""simple docstring"""





									_UpperCAmelCase					=      []
									_UpperCAmelCase					=      ''
									_UpperCAmelCase					=      False
									for token in tokens:
																# make sure that special tokens are not decoded using sentencepiece model
																if token in self.all_special_tokens:
																							if not prev_is_special:
																														out_string += " "
																							out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE						) + token
																							_UpperCAmelCase					=      True
																							_UpperCAmelCase					=      []
																else:
																							current_sub_tokens.append(_SCREAMING_SNAKE_CASE						)
																							_UpperCAmelCase					=      False
									out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE						)
									return out_string.strip()
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	List[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      [self.sep_token_id]
									_UpperCAmelCase					=      [self.cls_token_id]
									if token_ids_a is None:
																return cls + token_ids_a + sep
									return cls + token_ids_a + sep + token_ids_a + sep
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None      ,   _SCREAMING_SNAKE_CASE = False						)			->	List[int]:




									"""simple docstring"""





									if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_SCREAMING_SNAKE_CASE      ,   token_ids_a=_SCREAMING_SNAKE_CASE      ,   already_has_special_tokens=_SCREAMING_SNAKE_CASE						)

									if token_ids_a is not None:
																return [1] + ([0] * len(_SCREAMING_SNAKE_CASE						)) + [1] + ([0] * len(_SCREAMING_SNAKE_CASE						)) + [1]
									return [1] + ([0] * len(_SCREAMING_SNAKE_CASE						)) + [1]
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	List[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      [self.sep_token_id]
									_UpperCAmelCase					=      [self.cls_token_id]

									if token_ids_a is None:
																return len(cls + token_ids_a + sep						) * [0]
									return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	Tuple[str]:




									"""simple docstring"""





									if not os.path.isdir(_SCREAMING_SNAKE_CASE						):
																logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
																return
									_UpperCAmelCase					=      os.path.join(
									    _SCREAMING_SNAKE_CASE      ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

									if os.path.abspath(self.vocab_file						) != os.path.abspath(_SCREAMING_SNAKE_CASE						) and os.path.isfile(self.vocab_file						):
																copyfile(self.vocab_file      ,   _SCREAMING_SNAKE_CASE						)
									elif not os.path.isfile(self.vocab_file						):
																with open(_SCREAMING_SNAKE_CASE      ,   'wb'						) as fi:
																							_UpperCAmelCase					=      self.sp_model.serialized_model_proto()
																							fi.write(_SCREAMING_SNAKE_CASE						)

									return (out_vocab_file,)




# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


lowerCAmelCase__  :Dict             =							'''Run commands across TPU VMs for initial setup before running `accelerate launch`.'''

def 	lowerCAmelCase__			(      a__:		Optional[Any]=None	)  ->		List[Any]:


							'''simple docstring'''





							if subparsers is not None:
														_UpperCAmelCase					=      subparsers.add_parser('tpu-config'      ,  description=_description	)
							else:
														_UpperCAmelCase					=      argparse.ArgumentParser('Accelerate tpu-config command'      ,  description=_description	)
							# Core arguments
							_UpperCAmelCase					=      parser.add_argument_group(
							    'Config Arguments'      ,  'Arguments that can be configured through `accelerate config`.'	)
							config_args.add_argument(
							    '--config_file'      ,  type=a__      ,  default=a__      ,  help='Path to the config file to use for accelerate.'      ,  )
							config_args.add_argument(
							    '--tpu_name'      ,  default=a__      ,  help='The name of the TPU to use. If not specified, will use the TPU specified in the config file.'      ,  )
							config_args.add_argument(
							    '--tpu_zone'      ,  default=a__      ,  help='The zone of the TPU to use. If not specified, will use the zone specified in the config file.'      ,  )
							_UpperCAmelCase					=      parser.add_argument_group('TPU Arguments'      ,  'Arguments for options ran inside the TPU.'	)
							pod_args.add_argument(
							    '--use_alpha'      ,  action='store_true'      ,  help='Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.'      ,  )
							pod_args.add_argument(
							    '--command_file'      ,  default=a__      ,  help='The path to the file containing the commands to run on the pod on startup.'      ,  )
							pod_args.add_argument(
							    '--command'      ,  action='append'      ,  nargs='+'      ,  help='A command to run on the pod. Can be passed multiple times.'      ,  )
							pod_args.add_argument(
							    '--install_accelerate'      ,  action='store_true'      ,  help='Whether to install accelerate on the pod. Defaults to False.'      ,  )
							pod_args.add_argument(
							    '--accelerate_version'      ,  default='latest'      ,  help='The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify \'dev\' to install from GitHub.'      ,  )
							pod_args.add_argument(
							    '--debug'      ,  action='store_true'      ,  help='If set, will print the command that would be run instead of running it.'	)

							if subparsers is not None:
														parser.set_defaults(func=a__	)
							return parser

def 	lowerCAmelCase__			(      a__:		str	)  ->		Any:


							'''simple docstring'''





							_UpperCAmelCase					=      None

							# Get the default from the config file if it exists.
							if args.config_file is not None or os.path.isfile(a__	):
														_UpperCAmelCase					=      load_config_from_file(args.config_file	)
														if not args.command_file and defaults.command_file is not None and not args.command:
																					_UpperCAmelCase					=      defaults.command_file
														if not args.command and defaults.commands is not None:
																					_UpperCAmelCase					=      defaults.commands
														if not args.tpu_name:
																					_UpperCAmelCase					=      defaults.tpu_name
														if not args.tpu_zone:
																					_UpperCAmelCase					=      defaults.tpu_zone
							if args.accelerate_version == "dev":
														_UpperCAmelCase					=      'git+https://github.com/huggingface/accelerate.git'
							elif args.accelerate_version == "latest":
														_UpperCAmelCase					=      'accelerate -U'
							elif isinstance(parse(args.accelerate_version	)      ,  a__	):
														_UpperCAmelCase					=      F'''accelerate=={args.accelerate_version}'''

							if not args.command_file and not args.command:
														raise ValueError('You must specify either a command file or a command to run on the pod.'	)

							if args.command_file:
														with open(args.command_file      ,  'r'	) as f:
																					_UpperCAmelCase					=      [f.read().splitlines()]

    # To turn list of lists into list of strings
							if isinstance(args.command[0]      ,  a__	):
														_UpperCAmelCase					=      [line for cmd in args.command for line in cmd]
							# Default to the shared folder and install accelerate
							_UpperCAmelCase					=      ['cd /usr/share']
							if args.install_accelerate:
														new_cmd += [F'''pip install {args.accelerate_version}''']
							new_cmd += args.command
							_UpperCAmelCase					=      '; '.join(a__	)

							# Then send it to gcloud
							# Eventually try to use google-api-core to do this instead of subprocess
							_UpperCAmelCase					=      ['gcloud']
							if args.use_alpha:
														cmd += ["alpha"]
							cmd += [
							    "compute",
							    "tpus",
							    "tpu-vm",
							    "ssh",
							    args.tpu_name,
							    "--zone",
							    args.tpu_zone,
							    "--command",
							    args.command,
							    "--worker",
							    "all",
							]
							if args.debug:
														print(F'''Running {" ".join(a__	)}'''	)
														return
							subprocess.run(a__	)
							print('Successfully setup pod.'	)







def 	lowerCAmelCase__			(      )  ->		Optional[Any]:


							'''simple docstring'''





							_UpperCAmelCase					=      tpu_command_parser()
							_UpperCAmelCase					=      parser.parse_args()

							tpu_command_launcher(a__	)
