



'''simple docstring'''





import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     AlbertForMaskedLM,
     AlbertForMultipleChoice,
     AlbertForPreTraining,
     AlbertForQuestionAnswering,
     AlbertForSequenceClassification,
     AlbertForTokenClassification,
     AlbertModel,
 )
 from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST






class 	lowerCAmelCase_:
  '''simple docstring'''




  def __init__(   self  ,__UpperCAmelCase  ,__UpperCAmelCase=13  ,__UpperCAmelCase=7  ,__UpperCAmelCase=True  ,__UpperCAmelCase=True  ,__UpperCAmelCase=True  ,__UpperCAmelCase=True  ,__UpperCAmelCase=99  ,__UpperCAmelCase=16  ,__UpperCAmelCase=36  ,__UpperCAmelCase=6  ,__UpperCAmelCase=6  ,__UpperCAmelCase=6  ,__UpperCAmelCase=37  ,__UpperCAmelCase="gelu"  ,__UpperCAmelCase=0.1  ,__UpperCAmelCase=0.1  ,__UpperCAmelCase=512  ,__UpperCAmelCase=16  ,__UpperCAmelCase=2  ,__UpperCAmelCase=0.0_2  ,__UpperCAmelCase=3  ,__UpperCAmelCase=4  ,__UpperCAmelCase=None  ,)   ->		Dict:
      lowerCAmelCase__			:  str											= parent
      lowerCAmelCase__			:  Any											= batch_size
      lowerCAmelCase__			:  int											= seq_length
      lowerCAmelCase__			:  Optional[Any]											= is_training
      lowerCAmelCase__			:  Union[str, Any]											= use_input_mask
      lowerCAmelCase__			:  Optional[Any]											= use_token_type_ids
      lowerCAmelCase__			:  Optional[Any]											= use_labels
      lowerCAmelCase__			:  str											= vocab_size
      lowerCAmelCase__			:  int											= embedding_size
      lowerCAmelCase__			:  Dict											= hidden_size
      lowerCAmelCase__			:  Union[str, Any]											= num_hidden_layers
      lowerCAmelCase__			:  Optional[int]											= num_hidden_groups
      lowerCAmelCase__			:  List[Any]											= num_attention_heads
      lowerCAmelCase__			:  str											= intermediate_size
      lowerCAmelCase__			:  Optional[int]											= hidden_act
      lowerCAmelCase__			:  List[Any]											= hidden_dropout_prob
      lowerCAmelCase__			:  Tuple											= attention_probs_dropout_prob
      lowerCAmelCase__			:  Optional[int]											= max_position_embeddings
      lowerCAmelCase__			:  Optional[Any]											= type_vocab_size
      lowerCAmelCase__			:  Any											= type_sequence_label_size
      lowerCAmelCase__			:  List[Any]											= initializer_range
      lowerCAmelCase__			:  str											= num_labels
      lowerCAmelCase__			:  List[Any]											= num_choices
      lowerCAmelCase__			:  Dict											= scope




  def 							UpperCAmelCase_     (   self				)   ->		Any:
      lowerCAmelCase__			:  List[str]											= ids_tensor([self.batch_size, self.seq_length]  ,self.vocab_size				)

      lowerCAmelCase__			:  Tuple											= None
      if self.use_input_mask:
          lowerCAmelCase__			:  Optional[Any]											= random_attention_mask([self.batch_size, self.seq_length]				)

      lowerCAmelCase__			:  Union[str, Any]											= None
      if self.use_token_type_ids:
          lowerCAmelCase__			:  str											= ids_tensor([self.batch_size, self.seq_length]  ,self.type_vocab_size				)

      lowerCAmelCase__			:  Optional[Any]											= None
      lowerCAmelCase__			:  Optional[Any]											= None
      lowerCAmelCase__			:  List[str]											= None
      if self.use_labels:
          lowerCAmelCase__			:  str											= ids_tensor([self.batch_size]  ,self.type_sequence_label_size				)
          lowerCAmelCase__			:  Dict											= ids_tensor([self.batch_size, self.seq_length]  ,self.num_labels				)
          lowerCAmelCase__			:  List[Any]											= ids_tensor([self.batch_size]  ,self.num_choices				)

      lowerCAmelCase__			:  Union[str, Any]											= self.get_config()

      return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




  def 							UpperCAmelCase_     (   self				)   ->		int:
      return AlbertConfig(
          vocab_size=self.vocab_size  ,hidden_size=self.hidden_size  ,num_hidden_layers=self.num_hidden_layers  ,num_attention_heads=self.num_attention_heads  ,intermediate_size=self.intermediate_size  ,hidden_act=self.hidden_act  ,hidden_dropout_prob=self.hidden_dropout_prob  ,attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,max_position_embeddings=self.max_position_embeddings  ,type_vocab_size=self.type_vocab_size  ,initializer_range=self.initializer_range  ,num_hidden_groups=self.num_hidden_groups  ,)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		str:
      lowerCAmelCase__			:  List[Any]											= AlbertModel(config=__UpperCAmelCase				)
      model.to(__UpperCAmelCase				)
      model.eval()
      lowerCAmelCase__			:  int											= model(__UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase				)
      lowerCAmelCase__			:  List[str]											= model(__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase				)
      lowerCAmelCase__			:  Optional[Any]											= model(__UpperCAmelCase				)
      self.parent.assertEqual(result.last_hidden_state.shape  ,(self.batch_size, self.seq_length, self.hidden_size)				)
      self.parent.assertEqual(result.pooler_output.shape  ,(self.batch_size, self.hidden_size)				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		str:
      lowerCAmelCase__			:  Optional[Any]											= AlbertForPreTraining(config=__UpperCAmelCase				)
      model.to(__UpperCAmelCase				)
      model.eval()
      lowerCAmelCase__			:  Dict											= model(
          __UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,labels=__UpperCAmelCase  ,sentence_order_label=__UpperCAmelCase  ,)
      self.parent.assertEqual(result.prediction_logits.shape  ,(self.batch_size, self.seq_length, self.vocab_size)				)
      self.parent.assertEqual(result.sop_logits.shape  ,(self.batch_size, config.num_labels)				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		Optional[Any]:
      lowerCAmelCase__			:  Optional[Any]											= AlbertForMaskedLM(config=__UpperCAmelCase				)
      model.to(__UpperCAmelCase				)
      model.eval()
      lowerCAmelCase__			:  Union[str, Any]											= model(__UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,labels=__UpperCAmelCase				)
      self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.vocab_size)				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		Optional[int]:
      lowerCAmelCase__			:  Any											= AlbertForQuestionAnswering(config=__UpperCAmelCase				)
      model.to(__UpperCAmelCase				)
      model.eval()
      lowerCAmelCase__			:  Dict											= model(
          __UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,start_positions=__UpperCAmelCase  ,end_positions=__UpperCAmelCase  ,)
      self.parent.assertEqual(result.start_logits.shape  ,(self.batch_size, self.seq_length)				)
      self.parent.assertEqual(result.end_logits.shape  ,(self.batch_size, self.seq_length)				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		int:
      lowerCAmelCase__			:  Tuple											= self.num_labels
      lowerCAmelCase__			:  Optional[Any]											= AlbertForSequenceClassification(__UpperCAmelCase				)
      model.to(__UpperCAmelCase				)
      model.eval()
      lowerCAmelCase__			:  Dict											= model(__UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,labels=__UpperCAmelCase				)
      self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.num_labels)				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		List[Any]:
      lowerCAmelCase__			:  Optional[Any]											= self.num_labels
      lowerCAmelCase__			:  int											= AlbertForTokenClassification(config=__UpperCAmelCase				)
      model.to(__UpperCAmelCase				)
      model.eval()
      lowerCAmelCase__			:  Dict											= model(__UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,labels=__UpperCAmelCase				)
      self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.num_labels)				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		int:
      lowerCAmelCase__			:  List[Any]											= self.num_choices
      lowerCAmelCase__			:  List[Any]											= AlbertForMultipleChoice(config=__UpperCAmelCase				)
      model.to(__UpperCAmelCase				)
      model.eval()
      lowerCAmelCase__			:  Any											= input_ids.unsqueeze(1				).expand(-1  ,self.num_choices  ,-1				).contiguous()
      lowerCAmelCase__			:  Union[str, Any]											= token_type_ids.unsqueeze(1				).expand(-1  ,self.num_choices  ,-1				).contiguous()
      lowerCAmelCase__			:  int											= input_mask.unsqueeze(1				).expand(-1  ,self.num_choices  ,-1				).contiguous()
      lowerCAmelCase__			:  Union[str, Any]											= model(
          __UpperCAmelCase  ,attention_mask=__UpperCAmelCase  ,token_type_ids=__UpperCAmelCase  ,labels=__UpperCAmelCase  ,)
      self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.num_choices)				)






  def 							UpperCAmelCase_     (   self				)   ->		Dict:
      lowerCAmelCase__			:  int											= self.prepare_config_and_inputs()
      (
          (
          lowerCAmelCase__
      )				,       (
          lowerCAmelCase__
      )				,       (
          lowerCAmelCase__
      )				,       (
          lowerCAmelCase__
      )				,       (
          lowerCAmelCase__
      )				,       (
          lowerCAmelCase__
      )				,       (
          lowerCAmelCase__
      )				,       
      )			:  Dict											= config_and_inputs
      lowerCAmelCase__			:  Any											= {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
      return config, inputs_dict







@require_torch
class 	lowerCAmelCase_(    SCREAMING_SNAKE_CASE_						,			SCREAMING_SNAKE_CASE_						,			unittest.TestCase					):
  '''simple docstring'''




  __lowercase							:				Union[str, Any]           =				(
      (
          AlbertModel,
          AlbertForPreTraining,
          AlbertForMaskedLM,
          AlbertForMultipleChoice,
          AlbertForSequenceClassification,
          AlbertForTokenClassification,
          AlbertForQuestionAnswering,
      )
      if is_torch_available()
      else ()
  )
  __lowercase							:				Tuple           =				(
      {
          '''feature-extraction''': AlbertModel,
          '''fill-mask''': AlbertForMaskedLM,
          '''question-answering''': AlbertForQuestionAnswering,
          '''text-classification''': AlbertForSequenceClassification,
          '''token-classification''': AlbertForTokenClassification,
          '''zero-shot''': AlbertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  __lowercase							:				Tuple           =				True




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase=False				)   ->		str:
      lowerCAmelCase__			:  Tuple											= super()._prepare_for_class(__UpperCAmelCase  ,__UpperCAmelCase  ,return_labels=__UpperCAmelCase				)

      if return_labels:
          if model_class in get_values(__UpperCAmelCase				):
              lowerCAmelCase__			:  Optional[int]											= torch.zeros(
                  (self.model_tester.batch_size, self.model_tester.seq_length)  ,dtype=torch.long  ,device=__UpperCAmelCase				)
              lowerCAmelCase__			:  Tuple											= torch.zeros(
                  self.model_tester.batch_size  ,dtype=torch.long  ,device=__UpperCAmelCase				)
      return inputs_dict




  def 							UpperCAmelCase_     (   self				)   ->		Optional[Any]:
      lowerCAmelCase__			:  List[Any]											= AlbertModelTester(self				)
      lowerCAmelCase__			:  Optional[Any]											= ConfigTester(self  ,config_class=__UpperCAmelCase  ,hidden_size=37				)




  def 							UpperCAmelCase_     (   self				)   ->		str:
      self.config_tester.run_common_tests()




  def 							UpperCAmelCase_     (   self				)   ->		Tuple:
      lowerCAmelCase__			:  Optional[Any]											= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*__UpperCAmelCase				)




  def 							UpperCAmelCase_     (   self				)   ->		int:
      lowerCAmelCase__			:  str											= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_pretraining(*__UpperCAmelCase				)




  def 							UpperCAmelCase_     (   self				)   ->		Optional[Any]:
      lowerCAmelCase__			:  str											= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_masked_lm(*__UpperCAmelCase				)




  def 							UpperCAmelCase_     (   self				)   ->		Optional[int]:
      lowerCAmelCase__			:  int											= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_multiple_choice(*__UpperCAmelCase				)




  def 							UpperCAmelCase_     (   self				)   ->		Tuple:
      lowerCAmelCase__			:  List[str]											= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*__UpperCAmelCase				)




  def 							UpperCAmelCase_     (   self				)   ->		Tuple:
      lowerCAmelCase__			:  Any											= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*__UpperCAmelCase				)




  def 							UpperCAmelCase_     (   self				)   ->		Tuple:
      lowerCAmelCase__			:  Any											= self.model_tester.prepare_config_and_inputs()
      for type in ["absolute", "relative_key", "relative_key_query"]:
          lowerCAmelCase__			:  Optional[Any]											= type
          self.model_tester.create_and_check_model(*__UpperCAmelCase				)






  @slow
  def 							UpperCAmelCase_     (   self				)   ->		Tuple:
      for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          lowerCAmelCase__			:  Optional[int]											= AlbertModel.from_pretrained(__UpperCAmelCase				)
          self.assertIsNotNone(__UpperCAmelCase				)







@require_torch
class 	lowerCAmelCase_(    unittest.TestCase					):
  '''simple docstring'''




  @slow
  def 							UpperCAmelCase_     (   self				)   ->		Any:
      lowerCAmelCase__			:  List[str]											= AlbertModel.from_pretrained("""albert-base-v2"""				)
      lowerCAmelCase__			:  Union[str, Any]											= torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]				)
      lowerCAmelCase__			:  Optional[Any]											= torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]				)
      with torch.no_grad():
          lowerCAmelCase__			:  Optional[Any]											= model(__UpperCAmelCase  ,attention_mask=__UpperCAmelCase				)[0]
      lowerCAmelCase__			:  Tuple											= torch.Size((1, 11, 768)				)
      self.assertEqual(output.shape  ,__UpperCAmelCase				)
      lowerCAmelCase__			:  Optional[int]											= torch.tensor(
          [[[-0.6_5_1_3, 1.5_0_3_5, -0.2_7_6_6], [-0.6_5_1_5, 1.5_0_4_6, -0.2_7_8_0], [-0.6_5_1_2, 1.5_0_4_9, -0.2_7_8_4]]]				)

      self.assertTrue(torch.allclose(output[:, 1:4, 1:4]  ,__UpperCAmelCase  ,atol=1E-4				)				)







'''simple docstring'''





import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase										=     logging.get_logger(__name__)

_lowerCAmelCase										=     {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_lowerCAmelCase										=     {
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

_lowerCAmelCase										=     {
    '''ctrl''': 256,
}

_lowerCAmelCase										=     {
    '''Pregnancy''': 16_8629,
    '''Christianity''': 7675,
    '''Explain''': 10_6423,
    '''Fitness''': 6_3440,
    '''Saving''': 6_3163,
    '''Ask''': 2_7171,
    '''Ass''': 9_5985,
    '''Joke''': 16_3509,
    '''Questions''': 4_5622,
    '''Thoughts''': 4_9605,
    '''Retail''': 5_2342,
    '''Feminism''': 16_4338,
    '''Writing''': 1_1992,
    '''Atheism''': 19_2263,
    '''Netflix''': 4_8616,
    '''Computing''': 3_9639,
    '''Opinion''': 4_3213,
    '''Alone''': 4_4967,
    '''Funny''': 5_8917,
    '''Gaming''': 4_0358,
    '''Human''': 4088,
    '''India''': 1331,
    '''Joker''': 7_7138,
    '''Diet''': 3_6206,
    '''Legal''': 1_1859,
    '''Norman''': 4939,
    '''Tip''': 7_2689,
    '''Weight''': 5_2343,
    '''Movies''': 4_6273,
    '''Running''': 2_3425,
    '''Science''': 2090,
    '''Horror''': 3_7793,
    '''Confession''': 6_0572,
    '''Finance''': 1_2250,
    '''Politics''': 1_6360,
    '''Scary''': 19_1985,
    '''Support''': 1_2654,
    '''Technologies''': 3_2516,
    '''Teenage''': 6_6160,
    '''Event''': 3_2769,
    '''Learned''': 6_7460,
    '''Notion''': 18_2770,
    '''Wikipedia''': 3_7583,
    '''Books''': 6665,
    '''Extract''': 7_6050,
    '''Confessions''': 10_2701,
    '''Conspiracy''': 7_5932,
    '''Links''': 6_3674,
    '''Narcissus''': 15_0425,
    '''Relationship''': 5_4766,
    '''Relationships''': 13_4796,
    '''Reviews''': 4_1671,
    '''News''': 4256,
    '''Translation''': 2_6820,
    '''multilingual''': 12_8406,
}
def        _SCREAMING_SNAKE_CASE  (	UpperCamelCase      ):



    """simple docstring"""





    lowerCAmelCase__			:  Dict											= set()
    lowerCAmelCase__			:  List[str]											= word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)      )
        lowerCAmelCase__			:  str											= char

    lowerCAmelCase__			:  int											= set(UpperCamelCase      )
    return pairs






class 	lowerCAmelCase_(    SCREAMING_SNAKE_CASE_					):
  '''simple docstring'''




  __lowercase							:				Optional[int]           =				VOCAB_FILES_NAMES
  __lowercase							:				Union[str, Any]           =				PRETRAINED_VOCAB_FILES_MAP
  __lowercase							:				List[Any]           =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowercase							:				str           =				CONTROL_CODES




  def __init__(   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase="<unk>"  ,**__UpperCAmelCase				)   ->		Optional[Any]:
      super().__init__(unk_token=__UpperCAmelCase  ,**__UpperCAmelCase				)

      with open(__UpperCAmelCase  ,encoding="""utf-8"""				) as vocab_handle:
          lowerCAmelCase__			:  List[Any]											= json.load(__UpperCAmelCase				)
      lowerCAmelCase__			:  str											= {v: k for k, v in self.encoder.items()}
      with open(__UpperCAmelCase  ,encoding="""utf-8"""				) as merges_handle:
          lowerCAmelCase__			:  Any											= merges_handle.read().split("""\n"""				)[1:-1]
      lowerCAmelCase__			:  Optional[Any]											= [tuple(merge.split()				) for merge in merges]
      lowerCAmelCase__			:  Tuple											= dict(zip(__UpperCAmelCase  ,range(len(__UpperCAmelCase				)				)				)				)
      lowerCAmelCase__			:  int											= {}




  @property
  def 							UpperCAmelCase_     (   self				)   ->		List[Any]:
      return len(self.encoder				)




  def 							UpperCAmelCase_     (   self				)   ->		Optional[int]:
      return dict(self.encoder  ,**self.added_tokens_encoder				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Optional[Any]:
      if token in self.cache:
          return self.cache[token]
      lowerCAmelCase__			:  int											= tuple(__UpperCAmelCase				)
      lowerCAmelCase__			:  str											= tuple(list(word[:-1]				) + [word[-1] + """</w>"""]				)
      lowerCAmelCase__			:  Tuple											= get_pairs(__UpperCAmelCase				)

      if not pairs:
          return token

      while True:
          lowerCAmelCase__			:  Tuple											= min(__UpperCAmelCase  ,key=lambda __UpperCAmelCase				: self.bpe_ranks.get(__UpperCAmelCase  ,float("""inf"""				)				)				)
          if bigram not in self.bpe_ranks:
              break
          lowerCAmelCase__				,       lowerCAmelCase__			:  Optional[Any]											= bigram
          lowerCAmelCase__			:  Dict											= []
          lowerCAmelCase__			:  int											= 0
          while i < len(__UpperCAmelCase				):
              try:
                  lowerCAmelCase__			:  Any											= word.index(__UpperCAmelCase  ,__UpperCAmelCase				)
              except ValueError:
                  new_word.extend(word[i:]				)
                  break
              else:
                  new_word.extend(word[i:j]				)
                  lowerCAmelCase__			:  Any											= j

              if word[i] == first and i < len(__UpperCAmelCase				) - 1 and word[i + 1] == second:
                  new_word.append(first + second				)
                  i += 2
              else:
                  new_word.append(word[i]				)
                  i += 1
          lowerCAmelCase__			:  Any											= tuple(__UpperCAmelCase				)
          lowerCAmelCase__			:  Tuple											= new_word
          if len(__UpperCAmelCase				) == 1:
              break
          else:
              lowerCAmelCase__			:  List[str]											= get_pairs(__UpperCAmelCase				)
      lowerCAmelCase__			:  List[Any]											= """@@ """.join(__UpperCAmelCase				)
      lowerCAmelCase__			:  int											= word[:-4]
      lowerCAmelCase__			:  Optional[Any]											= word
      return word




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Union[str, Any]:
      lowerCAmelCase__			:  List[Any]											= []

      lowerCAmelCase__			:  Optional[int]											= re.findall(R"""\S+\n?"""  ,__UpperCAmelCase				)

      for token in words:
          split_tokens.extend(list(self.bpe(__UpperCAmelCase				).split(""" """				)				)				)
      return split_tokens




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Tuple:
      return self.encoder.get(__UpperCAmelCase  ,self.encoder.get(self.unk_token				)				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Optional[int]:
      return self.decoder.get(__UpperCAmelCase  ,self.unk_token				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Dict:
      lowerCAmelCase__			:  Optional[int]											= """ """.join(__UpperCAmelCase				).replace("""@@ """  ,""""""				).strip()
      return out_string






  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase = None				)   ->		Tuple[str]:
      if not os.path.isdir(__UpperCAmelCase				):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
          return
      lowerCAmelCase__			:  Tuple											= os.path.join(
          __UpperCAmelCase  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
      lowerCAmelCase__			:  str											= os.path.join(
          __UpperCAmelCase  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]				)

      with open(__UpperCAmelCase  ,"""w"""  ,encoding="""utf-8"""				) as f:
          f.write(json.dumps(self.encoder  ,indent=2  ,sort_keys=__UpperCAmelCase  ,ensure_ascii=__UpperCAmelCase				) + """\n"""				)

      lowerCAmelCase__			:  Optional[int]											= 0
      with open(__UpperCAmelCase  ,"""w"""  ,encoding="""utf-8"""				) as writer:
          writer.write("""#version: 0.2\n"""				)
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,key=lambda __UpperCAmelCase				: kv[1]				):
              if index != token_index:
                  logger.warning(
                      F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                      """ Please check that the tokenizer is not corrupted!"""				)
                  lowerCAmelCase__			:  List[str]											= token_index
              writer.write(""" """.join(__UpperCAmelCase				) + """\n"""				)
              index += 1

      return vocab_file, merge_file

  # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
  #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
  #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
  #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
  #     return ''.join(tokens_generated_so_far)


