


"""simple docstring"""
import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


__SCREAMING_SNAKE_CASE							:      Optional[Any]						=      logging.getLogger(__name__)





@dataclass
class        __A       :





				'''simple docstring'''

				__lowercase:						str
				__lowercase:						List[str]
				__lowercase:						Optional[List[str]]





@dataclass
class        __A       :





				'''simple docstring'''

				__lowercase:						List[int]
				__lowercase:						List[int]
				__lowercase:						Optional[List[int]]							 =      None
				__lowercase:						Optional[List[int]]							 =      None





class        __A       (snake_case__):





				'''simple docstring'''

				__lowercase:						Union[str, Any]							 =      """train"""
				__lowercase:						int							 =      """dev"""
				__lowercase:						Optional[int]							 =      """test"""





class        __A       :





				'''simple docstring'''

				@staticmethod
				def 						lowerCAmelCase    (			UpperCAmelCase_							: Optional[int]						,    UpperCAmelCase_							: Union[Split, str]    )   ->List[InputExample]:



											"""simple docstring"""



											raise NotImplementedError



				@staticmethod
				def 						lowerCAmelCase    (			UpperCAmelCase_							: str    )   ->List[str]:



											"""simple docstring"""



											raise NotImplementedError



				@staticmethod
				def 						lowerCAmelCase    (			UpperCAmelCase_							: List[InputExample]						,    UpperCAmelCase_							: List[str]						,    UpperCAmelCase_							: int						,    UpperCAmelCase_							: PreTrainedTokenizer						,    UpperCAmelCase_							: Dict=False						,    UpperCAmelCase_							: Optional[int]="[CLS]"						,    UpperCAmelCase_							: int=1						,    UpperCAmelCase_							: Any="[SEP]"						,    UpperCAmelCase_							: str=False						,    UpperCAmelCase_							: Optional[int]=False						,    UpperCAmelCase_							: List[Any]=0						,    UpperCAmelCase_							: List[Any]=0						,    UpperCAmelCase_							: Optional[Any]=-100						,    UpperCAmelCase_							: List[Any]=0						,    UpperCAmelCase_							: Union[str, Any]=True						,    )   ->List[InputFeatures]:



											"""simple docstring"""



											snake_case_          =						{label: i for i, label in enumerate(UpperCAmelCase_    )}

											snake_case_          =						[]
											for ex_index, example in enumerate(UpperCAmelCase_    ):
																		if ex_index % 10_000 == 0:
																									logger.info("""Writing example %d of %d"""						,    UpperCAmelCase_						,    len(UpperCAmelCase_    )    )

																		snake_case_          =						[]
																		snake_case_          =						[]
																		for word, label in zip(example.words						,    example.labels    ):
																									snake_case_          =						tokenizer.tokenize(UpperCAmelCase_    )

																									# bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
																									if len(UpperCAmelCase_    ) > 0:
																																tokens.extend(UpperCAmelCase_    )
																																# Use the real label id for the first token of the word, and padding ids for the remaining tokens
																																label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(UpperCAmelCase_    ) - 1)    )

            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
																		snake_case_          =						tokenizer.num_special_tokens_to_add()
																		if len(UpperCAmelCase_    ) > max_seq_length - special_tokens_count:
																									snake_case_          =						tokens[: (max_seq_length - special_tokens_count)]
																									snake_case_          =						label_ids[: (max_seq_length - special_tokens_count)]

																		# The convention in BERT is:
																		# (a) For sequence pairs:
																		#  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
																		#  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
																		# (b) For single sequences:
																		#  tokens:   [CLS] the dog is hairy . [SEP]
																		#  type_ids:   0   0   0   0  0     0   0
																		#
																		# Where "type_ids" are used to indicate whether this is the first
																		# sequence or the second sequence. The embedding vectors for `type=0` and
																		# `type=1` were learned during pre-training and are added to the wordpiece
																		# embedding vector (and position vector). This is not *strictly* necessary
																		# since the [SEP] token unambiguously separates the sequences, but it makes
																		# it easier for the model to learn the concept of sequences.
																		#
																		# For classification tasks, the first vector (corresponding to [CLS]) is
																		# used as the "sentence vector". Note that this only makes sense because
																		# the entire model is fine-tuned.
																		tokens += [sep_token]
																		label_ids += [pad_token_label_id]
																		if sep_token_extra:
																									# roberta uses an extra separator b/w pairs of sentences
																									tokens += [sep_token]
																									label_ids += [pad_token_label_id]
																		snake_case_          =						[sequence_a_segment_id] * len(UpperCAmelCase_    )

																		if cls_token_at_end:
																									tokens += [cls_token]
																									label_ids += [pad_token_label_id]
																									segment_ids += [cls_token_segment_id]
																		else:
																									snake_case_          =						[cls_token] + tokens
																									snake_case_          =						[pad_token_label_id] + label_ids
																									snake_case_          =						[cls_token_segment_id] + segment_ids

																		snake_case_          =						tokenizer.convert_tokens_to_ids(UpperCAmelCase_    )

																		# The mask has 1 for real tokens and 0 for padding tokens. Only real
																		# tokens are attended to.
																		snake_case_          =						[1 if mask_padding_with_zero else 0] * len(UpperCAmelCase_    )

																		# Zero-pad up to the sequence length.
																		snake_case_          =						max_seq_length - len(UpperCAmelCase_    )
																		if pad_on_left:
																									snake_case_          =						([pad_token] * padding_length) + input_ids
																									snake_case_          =						([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
																									snake_case_          =						([pad_token_segment_id] * padding_length) + segment_ids
																									snake_case_          =						([pad_token_label_id] * padding_length) + label_ids
																		else:
																									input_ids += [pad_token] * padding_length
																									input_mask += [0 if mask_padding_with_zero else 1] * padding_length
																									segment_ids += [pad_token_segment_id] * padding_length
																									label_ids += [pad_token_label_id] * padding_length

																		assert len(UpperCAmelCase_    ) == max_seq_length
																		assert len(UpperCAmelCase_    ) == max_seq_length
																		assert len(UpperCAmelCase_    ) == max_seq_length
																		assert len(UpperCAmelCase_    ) == max_seq_length

																		if ex_index < 5:
																									logger.info("""*** Example ***"""    )
																									logger.info("""guid: %s"""						,    example.guid    )
																									logger.info("""tokens: %s"""						,    """ """.join([str(UpperCAmelCase_    ) for x in tokens]    )    )
																									logger.info("""input_ids: %s"""						,    """ """.join([str(UpperCAmelCase_    ) for x in input_ids]    )    )
																									logger.info("""input_mask: %s"""						,    """ """.join([str(UpperCAmelCase_    ) for x in input_mask]    )    )
																									logger.info("""segment_ids: %s"""						,    """ """.join([str(UpperCAmelCase_    ) for x in segment_ids]    )    )
																									logger.info("""label_ids: %s"""						,    """ """.join([str(UpperCAmelCase_    ) for x in label_ids]    )    )

																		if "token_type_ids" not in tokenizer.model_input_names:
																									snake_case_          =						None

																		features.append(
																		    InputFeatures(
																		        input_ids=UpperCAmelCase_						,    attention_mask=UpperCAmelCase_						,    token_type_ids=UpperCAmelCase_						,    label_ids=UpperCAmelCase_    )    )
											return features


if is_torch_available():
				import torch
				from torch import nn
				from torch.utils.data import Dataset





				class        __A       (snake_case__):





								'''simple docstring'''

								__lowercase:						List[InputFeatures]
								__lowercase:						int							 =      nn.CrossEntropyLoss().ignore_index



								def __init__(			self							: Union[str, Any]						,    UpperCAmelCase_							: TokenClassificationTask						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: PreTrainedTokenizer						,    UpperCAmelCase_							: List[str]						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: Optional[int] = None						,    UpperCAmelCase_							: Any=False						,    UpperCAmelCase_							: Split = Split.train						,    )   ->int:



															"""simple docstring"""



															snake_case_          =						os.path.join(
															    UpperCAmelCase_						,    """cached_{}_{}_{}""".format(mode.value						,    tokenizer.__class__.__name__						,    str(UpperCAmelCase_    )    )						,    )

															# Make sure only the first process in distributed training processes the dataset,
															# and the others will use the cache.
															snake_case_          =						cached_features_file + """.lock"""
															with FileLock(UpperCAmelCase_    ):
																						if os.path.exists(UpperCAmelCase_    ) and not overwrite_cache:
																													logger.info(F"""Loading features from cached file {cached_features_file}"""    )
																													snake_case_          =						torch.load(UpperCAmelCase_    )
																						else:
																													logger.info(F"""Creating features from dataset file at {data_dir}"""    )
																													snake_case_          =						token_classification_task.read_examples_from_file(UpperCAmelCase_						,    UpperCAmelCase_    )
																													# TODO clean up all this to leverage built-in features of tokenizers
																													snake_case_          =						token_classification_task.convert_examples_to_features(
																													    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    cls_token_at_end=bool(model_type in ["""xlnet"""]    )						,    cls_token=tokenizer.cls_token						,    cls_token_segment_id=2 if model_type in ["""xlnet"""] else 0						,    sep_token=tokenizer.sep_token						,    sep_token_extra=UpperCAmelCase_						,    pad_on_left=bool(tokenizer.padding_side == """left"""    )						,    pad_token=tokenizer.pad_token_id						,    pad_token_segment_id=tokenizer.pad_token_type_id						,    pad_token_label_id=self.pad_token_label_id						,    )
																													logger.info(F"""Saving features into cached file {cached_features_file}"""    )
																													torch.save(self.features						,    UpperCAmelCase_    )



								def __len__(			self							: Optional[int]    )   ->Tuple:



															"""simple docstring"""



															return len(self.features    )



								def __getitem__(			self							: Dict						,    UpperCAmelCase_							: Union[str, Any]    )   ->InputFeatures:



															"""simple docstring"""



															return self.features[i]


if is_tf_available():
				import tensorflow as tf





				class        __A       :





								'''simple docstring'''

								__lowercase:						List[InputFeatures]
								__lowercase:						int							 =      -1_00



								def __init__(			self							: Optional[Any]						,    UpperCAmelCase_							: TokenClassificationTask						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: PreTrainedTokenizer						,    UpperCAmelCase_							: List[str]						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: Optional[int] = None						,    UpperCAmelCase_							: List[Any]=False						,    UpperCAmelCase_							: Split = Split.train						,    )   ->Dict:



															"""simple docstring"""



															snake_case_          =						token_classification_task.read_examples_from_file(UpperCAmelCase_						,    UpperCAmelCase_    )
															# TODO clean up all this to leverage built-in features of tokenizers
															snake_case_          =						token_classification_task.convert_examples_to_features(
															    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    UpperCAmelCase_						,    cls_token_at_end=bool(model_type in ["""xlnet"""]    )						,    cls_token=tokenizer.cls_token						,    cls_token_segment_id=2 if model_type in ["""xlnet"""] else 0						,    sep_token=tokenizer.sep_token						,    sep_token_extra=UpperCAmelCase_						,    pad_on_left=bool(tokenizer.padding_side == """left"""    )						,    pad_token=tokenizer.pad_token_id						,    pad_token_segment_id=tokenizer.pad_token_type_id						,    pad_token_label_id=self.pad_token_label_id						,    )

															def gen():
																						for ex in self.features:
																													if ex.token_type_ids is None:
																																				yield (
																																				    {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
																																				    ex.label_ids,
																																				)
																													else:
																																				yield (
																																				    {
																																				        "input_ids": ex.input_ids,
																																				        "attention_mask": ex.attention_mask,
																																				        "token_type_ids": ex.token_type_ids,
																																				    },
																																				    ex.label_ids,
																																				)

															if "token_type_ids" not in tokenizer.model_input_names:
																						snake_case_          =						tf.data.Dataset.from_generator(
																						    UpperCAmelCase_						,    ({"""input_ids""": tf.intaa, """attention_mask""": tf.intaa}, tf.intaa)						,    (
																						        {"""input_ids""": tf.TensorShape([None]    ), """attention_mask""": tf.TensorShape([None]    )},
																						        tf.TensorShape([None]    ),
																						    )						,    )
															else:
																						snake_case_          =						tf.data.Dataset.from_generator(
																						    UpperCAmelCase_						,    ({"""input_ids""": tf.intaa, """attention_mask""": tf.intaa, """token_type_ids""": tf.intaa}, tf.intaa)						,    (
																						        {
																						            """input_ids""": tf.TensorShape([None]    ),
																						            """attention_mask""": tf.TensorShape([None]    ),
																						            """token_type_ids""": tf.TensorShape([None]    ),
																						        },
																						        tf.TensorShape([None]    ),
																						    )						,    )



								def 						lowerCAmelCase    (			self							: str    )   ->List[str]:



															"""simple docstring"""



															snake_case_          =						self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features    )    )    )

															return self.dataset



								def __len__(			self							: str    )   ->List[str]:



															"""simple docstring"""



															return len(self.features    )



								def __getitem__(			self							: List[Any]						,    UpperCAmelCase_							: Optional[Any]    )   ->InputFeatures:



															"""simple docstring"""



															return self.features[i]







"""simple docstring"""
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__SCREAMING_SNAKE_CASE							:      Dict						=      logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							:      Tuple						=      {
    'google/mobilenet_v1_1.0_224': 'https://huggingface.co/google/mobilenet_v1_1.0_224/resolve/main/config.json',
    'google/mobilenet_v1_0.75_192': 'https://huggingface.co/google/mobilenet_v1_0.75_192/resolve/main/config.json',
    # See all MobileNetV1 models at https://huggingface.co/models?filter=mobilenet_v1
}





class        __A       (snake_case__):





							'''simple docstring'''

							__lowercase:						List[Any]							 =      """mobilenet_v1"""



							def __init__(			self							: Union[str, Any]						,    UpperCAmelCase_							: str=3						,    UpperCAmelCase_							: List[Any]=224						,    UpperCAmelCase_							: List[Any]=1.0						,    UpperCAmelCase_							: Any=8						,    UpperCAmelCase_							: int="relu6"						,    UpperCAmelCase_							: str=True						,    UpperCAmelCase_							: List[str]=0.999						,    UpperCAmelCase_							: Optional[Any]=0.02						,    UpperCAmelCase_							: List[Any]=0.001						,    **UpperCAmelCase_							: Any						,    )   ->Union[str, Any]:



														"""simple docstring"""



														super().__init__(**UpperCAmelCase_    )

														if depth_multiplier <= 0:
																					raise ValueError("""depth_multiplier must be greater than zero."""    )

														snake_case_          =						num_channels
														snake_case_          =						image_size
														snake_case_          =						depth_multiplier
														snake_case_          =						min_depth
														snake_case_          =						hidden_act
														snake_case_          =						tf_padding
														snake_case_          =						classifier_dropout_prob
														snake_case_          =						initializer_range
														snake_case_          =						layer_norm_eps





class        __A       (snake_case__):





							'''simple docstring'''

							__lowercase:						int							 =      version.parse("""1.11""")



							@property
							def 						lowerCAmelCase    (			self							: Union[str, Any]    )   ->Mapping[str, Mapping[int, str]]:



														"""simple docstring"""



														return OrderedDict([("""pixel_values""", {0: """batch"""})]    )



							@property
							def 						lowerCAmelCase    (			self							: int    )   ->Mapping[str, Mapping[int, str]]:



														"""simple docstring"""



														if self.task == "image-classification":
																					return OrderedDict([("""logits""", {0: """batch"""})]    )
														else:
																					return OrderedDict([("""last_hidden_state""", {0: """batch"""}), ("""pooler_output""", {0: """batch"""})]    )



							@property
							def 						lowerCAmelCase    (			self							: int    )   ->float:



														"""simple docstring"""



														return 1E-4



