"""simple docstring"""
import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


lowercase__ 	=     logging.getLogger(__name__)




@dataclass(frozen=__lowerCAmelCase    )
class        __snake_case	:
      a__				=		42
      a__				=		42
      a__				=		None
      a__				=		None
      a__				=		None







@dataclass(frozen=__lowerCAmelCase    )
class        __snake_case	:
      a__				=		42
      a__				=		None
      a__				=		None
      a__				=		None
      a__				=		None


if is_torch_available():
 import torch
 from torch.utils.data import Dataset




 class        __snake_case	(					__lowerCAmelCase    ):
       a__				=		42



       def __init__(						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase = None   ,      lowercase=False   ,      lowercase = False   ,      )     ->      str:





         '''simple docstring'''





         a__:					Tuple      							=     hans_processors[task]()

         a__:					Optional[int]      							=     os.path.join(
             lowercase   ,      'cached_{}_{}_{}_{}'.format(
                 'dev' if evaluate else 'train'   ,      tokenizer.__class__.__name__   ,      str(lowercase)   ,      lowercase   ,      )   ,      )
         a__:					Optional[int]      							=     processor.get_labels()
         if tokenizer.__class__ in (
             RobertaTokenizer,
             RobertaTokenizerFast,
             XLMRobertaTokenizer,
             BartTokenizer,
             BartTokenizerFast,
         ):
           # HACK(label indices are swapped in RoBERTa pretrained model)
           a__							,   a__:					int      							=     label_list[2], label_list[1]
         a__:					Optional[Any]      							=     label_list

         # Make sure only the first process in distributed training processes the dataset,
         # and the others will use the cache.
         a__:					List[str]      							=     cached_features_file + '.lock'
         with FileLock(lowercase):
           if os.path.exists(lowercase) and not overwrite_cache:
             logger.info(f'Loading features from cached file {cached_features_file}')
             a__:					Union[str, Any]      							=     torch.load(lowercase)
           else:
             logger.info(f'Creating features from dataset file at {data_dir}')

             a__:					Dict      							=     (
                 processor.get_dev_examples(lowercase) if evaluate else processor.get_train_examples(lowercase)
             )

             logger.info('Training examples: %s'   ,      len(lowercase))
             a__:					int      							=     hans_convert_examples_to_features(lowercase   ,      lowercase   ,      lowercase   ,      lowercase)
             logger.info('Saving features into cached file %s'   ,      lowercase)
             torch.save(self.features   ,      lowercase)



       def __len__(						self)     ->      int:





         '''simple docstring'''





         return len(self.features)



       def __getitem__(						self   ,      lowercase)     ->      InputFeatures:





         '''simple docstring'''





         return self.features[i]



       def   lowerCamelCase_       (						self)     ->      int:





         '''simple docstring'''





         return self.label_list


if is_tf_available():
 import tensorflow as tf




 class        __snake_case	:
       a__				=		42



       def __init__(						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase = 1_28   ,      lowercase=False   ,      lowercase = False   ,      )     ->      Dict:





         '''simple docstring'''





         a__:					Union[str, Any]      							=     hans_processors[task]()
         a__:					str      							=     processor.get_labels()
         if tokenizer.__class__ in (
             RobertaTokenizer,
             RobertaTokenizerFast,
             XLMRobertaTokenizer,
             BartTokenizer,
             BartTokenizerFast,
         ):
           # HACK(label indices are swapped in RoBERTa pretrained model)
           a__							,   a__:					int      							=     label_list[2], label_list[1]
         a__:					Optional[int]      							=     label_list

         a__:					str      							=     processor.get_dev_examples(lowercase) if evaluate else processor.get_train_examples(lowercase)
         a__:					Tuple      							=     hans_convert_examples_to_features(lowercase   ,      lowercase   ,      lowercase   ,      lowercase)

         def gen():
           for ex_index, ex in tqdm.tqdm(enumerate(self.features)   ,      desc='convert examples to features'):
             if ex_index % 1_00_00 == 0:
               logger.info('Writing example %d of %d' % (ex_index, len(lowercase)))

             yield (
                 {
                     "example_id": 0,
                     "input_ids": ex.input_ids,
                     "attention_mask": ex.attention_mask,
                     "token_type_ids": ex.token_type_ids,
                 },
                 ex.label,
             )

         a__:					Union[str, Any]      							=     tf.data.Dataset.from_generator(
             lowercase   ,      (
                 {
                     'example_id': tf.intaa,
                     'input_ids': tf.intaa,
                     'attention_mask': tf.intaa,
                     'token_type_ids': tf.intaa,
                 },
                 tf.intaa,
             )   ,      (
                 {
                     'example_id': tf.TensorShape([]),
                     'input_ids': tf.TensorShape([None, None]),
                     'attention_mask': tf.TensorShape([None, None]),
                     'token_type_ids': tf.TensorShape([None, None]),
                 },
                 tf.TensorShape([]),
             )   ,      )



       def   lowerCamelCase_       (						self)     ->      int:





         '''simple docstring'''





         return self.dataset



       def __len__(						self)     ->      int:





         '''simple docstring'''





         return len(self.features)



       def __getitem__(						self   ,      lowercase)     ->      InputFeatures:





         '''simple docstring'''





         return self.features[i]



       def   lowerCamelCase_       (						self)     ->      Tuple:





         '''simple docstring'''





         return self.label_list




class        __snake_case	(					__lowerCAmelCase    ):



      def   lowerCamelCase_       (						self   ,      lowercase)     ->      Optional[Any]:





        '''simple docstring'''





        return self._create_examples(self._read_tsv(os.path.join(lowercase   ,      'heuristics_train_set.txt'))   ,      'train')



      def   lowerCamelCase_       (						self   ,      lowercase)     ->      int:





        '''simple docstring'''





        return self._create_examples(self._read_tsv(os.path.join(lowercase   ,      'heuristics_evaluation_set.txt'))   ,      'dev')



      def   lowerCamelCase_       (						self)     ->      int:





        '''simple docstring'''





        return ["contradiction", "entailment", "neutral"]



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase)     ->      int:





        '''simple docstring'''





        a__:					Any      							=     []
        for i, line in enumerate(lowercase):
          if i == 0:
            continue
          a__:					Union[str, Any]      							=     '%s-%s' % (set_type, line[0])
          a__:					Union[str, Any]      							=     line[5]
          a__:					Optional[int]      							=     line[6]
          a__:					int      							=     line[7][2:] if line[7].startswith('ex') else line[7]
          a__:					int      							=     line[0]
          examples.append(InputExample(guid=lowercase   ,      text_a=lowercase   ,      text_b=lowercase   ,      label=lowercase   ,      pairID=lowercase))
        return examples







def 						__a       (     _SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					_SCREAMING_SNAKE_CASE		,					)     ->Optional[int]:
  a__:					Optional[Any]      							=     {label: i for i, label in enumerate(_SCREAMING_SNAKE_CASE       )}

  a__:					Any      							=     []
  for ex_index, example in tqdm.tqdm(enumerate(_SCREAMING_SNAKE_CASE       )		,					desc='convert examples to features'       ):
    if ex_index % 10000 == 0:
      logger.info('Writing example %d' % (ex_index)       )

    a__:					Any      							=     tokenizer(
        example.text_a		,					example.text_b		,					add_special_tokens=_SCREAMING_SNAKE_CASE		,					max_length=_SCREAMING_SNAKE_CASE		,					padding='max_length'		,					truncation=_SCREAMING_SNAKE_CASE		,					return_overflowing_tokens=_SCREAMING_SNAKE_CASE		,					)

    a__:					Union[str, Any]      							=     label_map[example.label] if example.label in label_map else 0

    a__:					int      							=     int(example.pairID       )

    features.append(InputFeatures(**_SCREAMING_SNAKE_CASE		,					label=_SCREAMING_SNAKE_CASE		,					pairID=_SCREAMING_SNAKE_CASE       )       )

  for i, example in enumerate(examples[:5]       ):
    logger.info('*** Example ***'       )
    logger.info(F'guid: {example}'       )
    logger.info(F'features: {features[i]}'       )

  return features


lowercase__ 	=     {
    'hans': 3,
}


lowercase__ 	=     {
    'hans': HansProcessor,
}



"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowercase__ 	=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase__ 	=     {
    'vocab_file': {
        'unc-nlp/lxmert-base-uncased': 'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'unc-nlp/lxmert-base-uncased': (
            'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'
        ),
    },
}

lowercase__ 	=     {
    'unc-nlp/lxmert-base-uncased': 512,
}

lowercase__ 	=     {
    'unc-nlp/lxmert-base-uncased': {'do_lower_case': True},
}




class        __snake_case	(					__lowerCAmelCase    ):
      a__				=		VOCAB_FILES_NAMES
      a__				=		PRETRAINED_VOCAB_FILES_MAP
      a__				=		PRETRAINED_INIT_CONFIGURATION
      a__				=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a__				=		LxmertTokenizer



      def __init__(						self   ,      lowercase=None   ,      lowercase=None   ,      lowercase=True   ,      lowercase="[UNK]"   ,      lowercase="[SEP]"   ,      lowercase="[PAD]"   ,      lowercase="[CLS]"   ,      lowercase="[MASK]"   ,      lowercase=True   ,      lowercase=None   ,      **lowercase   ,      )     ->      Dict:





        '''simple docstring'''





        super().__init__(
            lowercase   ,      tokenizer_file=lowercase   ,      do_lower_case=lowercase   ,      unk_token=lowercase   ,      sep_token=lowercase   ,      pad_token=lowercase   ,      cls_token=lowercase   ,      mask_token=lowercase   ,      tokenize_chinese_chars=lowercase   ,      strip_accents=lowercase   ,      **lowercase   ,      )

        a__:					Dict      							=     json.loads(self.backend_tokenizer.normalizer.__getstate__())
        if (
            normalizer_state.get('lowercase'   ,      lowercase) != do_lower_case
            or normalizer_state.get('strip_accents'   ,      lowercase) != strip_accents
            or normalizer_state.get('handle_chinese_chars'   ,      lowercase) != tokenize_chinese_chars
        ):
          a__:					int      							=     getattr(lowercase   ,      normalizer_state.pop('type'))
          a__:					Dict      							=     do_lower_case
          a__:					Dict      							=     strip_accents
          a__:					Optional[int]      							=     tokenize_chinese_chars
          a__:					List[Any]      							=     normalizer_class(**lowercase)

        a__:					Optional[int]      							=     do_lower_case



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase=None)     ->      Tuple:





        '''simple docstring'''





        a__:					Optional[Any]      							=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

        return output



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase = None)     ->      List[int]:





        '''simple docstring'''





        a__:					List[Any]      							=     [self.sep_token_id]
        a__:					List[Any]      							=     [self.cls_token_id]
        if token_ids_a is None:
          return len(cls + token_ids_a + sep) * [0]
        return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase = None)     ->      Tuple[str]:





        '''simple docstring'''





        a__:					List[Any]      							=     self._tokenizer.model.save(lowercase   ,      name=lowercase)
        return tuple(lowercase)


