





'''simple docstring'''



import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	def __init__(self       ,_lowerCamelCase = "▁"       ,_lowerCamelCase = True       ,_lowerCamelCase = "<unk>"       ,_lowerCamelCase = "</s>"       ,_lowerCamelCase = "<pad>"       ,)							->							Any:







				'''simple docstring'''





				__lowercase			  =							{
				    '''pad''': {'''id''': 0, '''token''': pad_token},
				    '''eos''': {'''id''': 1, '''token''': eos_token},
				    '''unk''': {'''id''': 2, '''token''': unk_token},
				}

				__lowercase			  =							[None] * len(self.special_tokens	)
				for token_dict in self.special_tokens.values():
							__lowercase			  =							token_dict['''token''']

				__lowercase			  =							Tokenizer(Unigram()	)

				__lowercase			  =							normalizers.Sequence(
				    [
				        normalizers.Nmt(),
				        normalizers.NFKC(),
				        normalizers.Replace(Regex(''' {2,}'''	)       ,''' '''	),
				        normalizers.Lowercase(),
				    ]	)
				__lowercase			  =							pre_tokenizers.Sequence(
				    [
				        pre_tokenizers.Metaspace(replacement=_lowerCamelCase       ,add_prefix_space=_lowerCamelCase	),
				        pre_tokenizers.Digits(individual_digits=_lowerCamelCase	),
				        pre_tokenizers.Punctuation(),
				    ]	)
				__lowercase			  =							decoders.Metaspace(replacement=_lowerCamelCase       ,add_prefix_space=_lowerCamelCase	)

				__lowercase			  =							TemplateProcessing(
				    single=f"$A {self.special_tokens['eos']['token']}"       ,special_tokens=[(self.special_tokens['''eos''']['''token'''], self.special_tokens['''eos''']['''id'''])]       ,)

				__lowercase			  =							{
				    '''model''': '''SentencePieceUnigram''',
				    '''replacement''': replacement,
				    '''add_prefix_space''': add_prefix_space,
				}

				super().__init__(_lowerCamelCase       ,_lowerCamelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = 8000       ,_lowerCamelCase = True       ,)							->							Optional[Any]:







				'''simple docstring'''





				__lowercase			  =							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase       ,special_tokens=self.special_tokens_list       ,show_progress=_lowerCamelCase       ,)

				if isinstance(_lowerCamelCase       ,_lowerCamelCase	):
							__lowercase			  =							[files]
				self._tokenizer.train(_lowerCamelCase       ,trainer=_lowerCamelCase	)

				self.add_unk_id()


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = 8000       ,_lowerCamelCase = True       ,)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase       ,special_tokens=self.special_tokens_list       ,show_progress=_lowerCamelCase       ,)

				self._tokenizer.train_from_iterator(_lowerCamelCase       ,trainer=_lowerCamelCase	)

				self.add_unk_id()


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							json.loads(self._tokenizer.to_str()	)

				__lowercase			  =							self.special_tokens['''unk''']['''id''']

				__lowercase			  =							Tokenizer.from_str(json.dumps(_lowerCamelCase	)	)












'''simple docstring'''



from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_SCREAMING_SNAKE_CASE   =      {'''configuration_focalnet''': ['''FOCALNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''FocalNetConfig''']}


try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_SCREAMING_SNAKE_CASE   =      [
	    '''FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''FocalNetForImageClassification''',
	    '''FocalNetForMaskedImageModeling''',
	    '''FocalNetBackbone''',
	    '''FocalNetModel''',
	    '''FocalNetPreTrainedModel''',
	]

if TYPE_CHECKING:
	from .configuration_focalnet import FOCALNET_PRETRAINED_CONFIG_ARCHIVE_MAP, FocalNetConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_focalnet import (
		    FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST,
		    FocalNetBackbone,
		    FocalNetForImageClassification,
		    FocalNetForMaskedImageModeling,
		    FocalNetModel,
		    FocalNetPreTrainedModel,
		)

else:
	import sys

	_SCREAMING_SNAKE_CASE   =      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





