



'''simple docstring'''





import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY






@is_pipeline_test
class 	lowerCAmelCase_(    unittest.TestCase					):
  '''simple docstring'''




  __lowercase							:				Tuple           =				MODEL_FOR_MASKED_LM_MAPPING
  __lowercase							:				str           =				TF_MODEL_FOR_MASKED_LM_MAPPING




  def 							UpperCAmelCase_     (   self				)   ->		Tuple:
      super().tearDown()
      # clean-up as much as possible GPU memory occupied by PyTorch
      gc.collect()
      if is_torch_available():
          import torch

          torch.cuda.empty_cache()




  @require_tf
  def 							UpperCAmelCase_     (   self				)   ->		Optional[int]:
      lowerCAmelCase__			:  Optional[int]											= pipeline(task="""fill-mask"""  ,model="""sshleifer/tiny-distilroberta-base"""  ,top_k=2  ,framework="""tf"""				)
      lowerCAmelCase__			:  Optional[int]											= unmasker("""My name is <mask>"""				)
      self.assertEqual(
          nested_simplify(_a  ,decimals=6				)  ,[
              {"""sequence""": """My name is grouped""", """score""": 2.1E-05, """token""": 3_8015, """token_str""": """ grouped"""},
              {"""sequence""": """My name is accuser""", """score""": 2.1E-05, """token""": 2_5506, """token_str""": """ accuser"""},
          ]  ,)

      lowerCAmelCase__			:  str											= unmasker("""The largest city in France is <mask>"""				)
      self.assertEqual(
          nested_simplify(_a  ,decimals=6				)  ,[
              {
                  """sequence""": """The largest city in France is grouped""",
                  """score""": 2.1E-05,
                  """token""": 3_8015,
                  """token_str""": """ grouped""",
              },
              {
                  """sequence""": """The largest city in France is accuser""",
                  """score""": 2.1E-05,
                  """token""": 2_5506,
                  """token_str""": """ accuser""",
              },
          ]  ,)

      lowerCAmelCase__			:  int											= unmasker("""My name is <mask>"""  ,targets=[""" Patrick""", """ Clara""", """ Teven"""]  ,top_k=3				)
      self.assertEqual(
          nested_simplify(_a  ,decimals=6				)  ,[
              {"""sequence""": """My name is Clara""", """score""": 2E-05, """token""": 1_3606, """token_str""": """ Clara"""},
              {"""sequence""": """My name is Patrick""", """score""": 2E-05, """token""": 3499, """token_str""": """ Patrick"""},
              {"""sequence""": """My name is Te""", """score""": 1.9E-05, """token""": 2941, """token_str""": """ Te"""},
          ]  ,)




  @require_torch
  def 							UpperCAmelCase_     (   self				)   ->		Any:
      lowerCAmelCase__			:  List[Any]											= pipeline(task="""fill-mask"""  ,model="""sshleifer/tiny-distilroberta-base"""  ,top_k=2  ,framework="""pt"""				)

      lowerCAmelCase__			:  Any											= unmasker("""My name is <mask>"""				)
      self.assertEqual(
          nested_simplify(_a  ,decimals=6				)  ,[
              {"""sequence""": """My name is Maul""", """score""": 2.2E-05, """token""": 3_5676, """token_str""": """ Maul"""},
              {"""sequence""": """My name isELS""", """score""": 2.2E-05, """token""": 1_6416, """token_str""": """ELS"""},
          ]  ,)

      lowerCAmelCase__			:  Tuple											= unmasker("""The largest city in France is <mask>"""				)
      self.assertEqual(
          nested_simplify(_a  ,decimals=6				)  ,[
              {
                  """sequence""": """The largest city in France is Maul""",
                  """score""": 2.2E-05,
                  """token""": 3_5676,
                  """token_str""": """ Maul""",
              },
              {"""sequence""": """The largest city in France isELS""", """score""": 2.2E-05, """token""": 1_6416, """token_str""": """ELS"""},
          ]  ,)

      lowerCAmelCase__			:  Any											= unmasker("""My name is <mask>"""  ,targets=[""" Patrick""", """ Clara""", """ Teven"""]  ,top_k=3				)
      self.assertEqual(
          nested_simplify(_a  ,decimals=6				)  ,[
              {"""sequence""": """My name is Patrick""", """score""": 2.1E-05, """token""": 3499, """token_str""": """ Patrick"""},
              {"""sequence""": """My name is Te""", """score""": 2E-05, """token""": 2941, """token_str""": """ Te"""},
              {"""sequence""": """My name is Clara""", """score""": 2E-05, """token""": 1_3606, """token_str""": """ Clara"""},
          ]  ,)

      lowerCAmelCase__			:  Union[str, Any]											= unmasker("""My name is <mask> <mask>"""  ,top_k=2				)

      self.assertEqual(
          nested_simplify(_a  ,decimals=6				)  ,[
              [
                  {
                      """score""": 2.2E-05,
                      """token""": 3_5676,
                      """token_str""": """ Maul""",
                      """sequence""": """<s>My name is Maul<mask></s>""",
                  },
                  {"""score""": 2.2E-05, """token""": 1_6416, """token_str""": """ELS""", """sequence""": """<s>My name isELS<mask></s>"""},
              ],
              [
                  {
                      """score""": 2.2E-05,
                      """token""": 3_5676,
                      """token_str""": """ Maul""",
                      """sequence""": """<s>My name is<mask> Maul</s>""",
                  },
                  {"""score""": 2.2E-05, """token""": 1_6416, """token_str""": """ELS""", """sequence""": """<s>My name is<mask>ELS</s>"""},
              ],
          ]  ,)




  @require_torch_gpu
  def 							UpperCAmelCase_     (   self				)   ->		Optional[Any]:
      lowerCAmelCase__			:  List[str]											= pipeline("""fill-mask"""  ,model="""hf-internal-testing/tiny-random-distilbert"""  ,device=0  ,framework="""pt"""				)

      # convert model to fp16
      pipe.model.half()

      lowerCAmelCase__			:  str											= pipe("""Paris is the [MASK] of France."""				)
      # We actually don't care about the result, we just want to make sure
      # it works, meaning the float16 tensor got casted back to float32
      # for postprocessing.
      self.assertIsInstance(_a  ,_a				)




  @slow
  @require_torch
  def 							UpperCAmelCase_     (   self				)   ->		Tuple:
      lowerCAmelCase__			:  Union[str, Any]											= pipeline(task="""fill-mask"""  ,model="""distilroberta-base"""  ,top_k=2  ,framework="""pt"""				)
      self.run_large_test(_a				)




  @slow
  @require_tf
  def 							UpperCAmelCase_     (   self				)   ->		Dict:
      lowerCAmelCase__			:  List[Any]											= pipeline(task="""fill-mask"""  ,model="""distilroberta-base"""  ,top_k=2  ,framework="""tf"""				)
      self.run_large_test(_a				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Optional[int]:
      lowerCAmelCase__			:  Tuple											= unmasker("""My name is <mask>"""				)
      self.assertEqual(
          nested_simplify(_a				)  ,[
              {"""sequence""": """My name is John""", """score""": 0.0_0_8, """token""": 610, """token_str""": """ John"""},
              {"""sequence""": """My name is Chris""", """score""": 0.0_0_7, """token""": 1573, """token_str""": """ Chris"""},
          ]  ,)
      lowerCAmelCase__			:  Dict											= unmasker("""The largest city in France is <mask>"""				)
      self.assertEqual(
          nested_simplify(_a				)  ,[
              {
                  """sequence""": """The largest city in France is Paris""",
                  """score""": 0.2_5_1,
                  """token""": 2201,
                  """token_str""": """ Paris""",
              },
              {
                  """sequence""": """The largest city in France is Lyon""",
                  """score""": 0.2_1_4,
                  """token""": 1_2790,
                  """token_str""": """ Lyon""",
              },
          ]  ,)

      lowerCAmelCase__			:  List[str]											= unmasker("""My name is <mask>"""  ,targets=[""" Patrick""", """ Clara""", """ Teven"""]  ,top_k=3				)
      self.assertEqual(
          nested_simplify(_a				)  ,[
              {"""sequence""": """My name is Patrick""", """score""": 0.0_0_5, """token""": 3499, """token_str""": """ Patrick"""},
              {"""sequence""": """My name is Clara""", """score""": 0.0_0_0, """token""": 1_3606, """token_str""": """ Clara"""},
              {"""sequence""": """My name is Te""", """score""": 0.0_0_0, """token""": 2941, """token_str""": """ Te"""},
          ]  ,)




  @require_torch
  def 							UpperCAmelCase_     (   self				)   ->		List[str]:
      lowerCAmelCase__			:  int											= pipeline(task="""fill-mask"""  ,model="""sshleifer/tiny-distilroberta-base"""  ,framework="""pt"""				)
      lowerCAmelCase__			:  Tuple											= None
      lowerCAmelCase__			:  Optional[int]											= None
      self.run_pipeline_test(_a  ,[]				)




  @require_tf
  def 							UpperCAmelCase_     (   self				)   ->		List[str]:
      lowerCAmelCase__			:  Dict											= pipeline(task="""fill-mask"""  ,model="""sshleifer/tiny-distilroberta-base"""  ,framework="""tf"""				)
      lowerCAmelCase__			:  Any											= None
      lowerCAmelCase__			:  Any											= None
      self.run_pipeline_test(_a  ,[]				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		Tuple:
      if tokenizer is None or tokenizer.mask_token_id is None:
          self.skipTest("""The provided tokenizer has no mask token, (probably reformer or wav2vec2)"""				)

      lowerCAmelCase__			:  int											= FillMaskPipeline(model=_a  ,tokenizer=_a				)
      lowerCAmelCase__			:  Optional[Any]											= [
          F"""This is another {tokenizer.mask_token} test""",
      ]
      return fill_masker, examples




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		int:
      lowerCAmelCase__			:  Union[str, Any]											= fill_masker.tokenizer
      lowerCAmelCase__			:  int											= fill_masker.model

      lowerCAmelCase__			:  int											= fill_masker(
          F"""This is a {tokenizer.mask_token}"""  ,)
      self.assertEqual(
          _a  ,[
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
          ]  ,)

      lowerCAmelCase__			:  List[Any]											= fill_masker([F"""This is a {tokenizer.mask_token}"""]				)
      self.assertEqual(
          _a  ,[
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
          ]  ,)

      lowerCAmelCase__			:  Tuple											= fill_masker([F"""This is a {tokenizer.mask_token}""", F"""Another {tokenizer.mask_token} great test."""]				)
      self.assertEqual(
          _a  ,[
              [
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              ],
              [
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              ],
          ]  ,)

      with self.assertRaises(_a				):
          fill_masker([None]				)
      # No mask_token is not supported
      with self.assertRaises(_a				):
          fill_masker("""This is"""				)

      self.run_test_top_k(_a  ,_a				)
      self.run_test_targets(_a  ,_a				)
      self.run_test_top_k_targets(_a  ,_a				)
      self.fill_mask_with_duplicate_targets_and_top_k(_a  ,_a				)
      self.fill_mask_with_multiple_masks(_a  ,_a				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		List[str]:
      lowerCAmelCase__			:  str											= tokenizer.get_vocab()
      lowerCAmelCase__			:  Any											= sorted(vocab.keys()				)[:2]
      # Pipeline argument
      lowerCAmelCase__			:  Tuple											= FillMaskPipeline(model=_a  ,tokenizer=_a  ,targets=_a				)
      lowerCAmelCase__			:  Union[str, Any]											= fill_masker(F"""This is a {tokenizer.mask_token}"""				)
      self.assertEqual(
          _a  ,[
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
          ]  ,)
      lowerCAmelCase__			:  str											= {vocab[el] for el in targets}
      self.assertEqual({el["""token"""] for el in outputs}  ,_a				)
      lowerCAmelCase__			:  int											= [tokenizer.decode([x]				) for x in target_ids]
      self.assertEqual({el["""token_str"""] for el in outputs}  ,set(_a				)				)

      # Call argument
      lowerCAmelCase__			:  Union[str, Any]											= FillMaskPipeline(model=_a  ,tokenizer=_a				)
      lowerCAmelCase__			:  Optional[int]											= fill_masker(F"""This is a {tokenizer.mask_token}"""  ,targets=_a				)
      self.assertEqual(
          _a  ,[
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
          ]  ,)
      lowerCAmelCase__			:  List[str]											= {vocab[el] for el in targets}
      self.assertEqual({el["""token"""] for el in outputs}  ,_a				)
      lowerCAmelCase__			:  Tuple											= [tokenizer.decode([x]				) for x in target_ids]
      self.assertEqual({el["""token_str"""] for el in outputs}  ,set(_a				)				)

      # Score equivalence
      lowerCAmelCase__			:  Union[str, Any]											= fill_masker(F"""This is a {tokenizer.mask_token}"""  ,targets=_a				)
      lowerCAmelCase__			:  Optional[int]											= [top_mask["token_str"] for top_mask in outputs]
      lowerCAmelCase__			:  Union[str, Any]											= [top_mask["score"] for top_mask in outputs]

      # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
      if set(_a				) == set(_a				):
          lowerCAmelCase__			:  Optional[Any]											= fill_masker(F"""This is a {tokenizer.mask_token}"""  ,targets=_a				)
          lowerCAmelCase__			:  Dict											= [top_mask["score"] for top_mask in unmasked_targets]
          self.assertEqual(nested_simplify(_a				)  ,nested_simplify(_a				)				)

      # Raises with invalid
      with self.assertRaises(_a				):
          lowerCAmelCase__			:  int											= fill_masker(F"""This is a {tokenizer.mask_token}"""  ,targets=[]				)
      # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
      if "" not in tokenizer.get_vocab():
          with self.assertRaises(_a				):
              lowerCAmelCase__			:  int											= fill_masker(F"""This is a {tokenizer.mask_token}"""  ,targets=[""""""]				)
          with self.assertRaises(_a				):
              lowerCAmelCase__			:  str											= fill_masker(F"""This is a {tokenizer.mask_token}"""  ,targets=""""""				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		Any:
      lowerCAmelCase__			:  str											= FillMaskPipeline(model=_a  ,tokenizer=_a  ,top_k=2				)
      lowerCAmelCase__			:  int											= fill_masker(F"""This is a {tokenizer.mask_token}"""				)
      self.assertEqual(
          _a  ,[
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
          ]  ,)

      lowerCAmelCase__			:  List[Any]											= FillMaskPipeline(model=_a  ,tokenizer=_a				)
      lowerCAmelCase__			:  List[str]											= fill_masker(F"""This is a {tokenizer.mask_token}"""  ,top_k=2				)
      self.assertEqual(
          _a  ,[
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
          ]  ,)
      self.assertEqual(nested_simplify(_a				)  ,nested_simplify(_a				)				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		Union[str, Any]:
      lowerCAmelCase__			:  Tuple											= tokenizer.get_vocab()
      lowerCAmelCase__			:  List[str]											= FillMaskPipeline(model=_a  ,tokenizer=_a				)

      # top_k=2, ntargets=3
      lowerCAmelCase__			:  int											= sorted(vocab.keys()				)[:3]
      lowerCAmelCase__			:  Any											= fill_masker(F"""This is a {tokenizer.mask_token}"""  ,top_k=2  ,targets=_a				)

      # If we use the most probably targets, and filter differently, we should still
      # have the same results
      lowerCAmelCase__			:  Optional[Any]											= [el["token_str"] for el in sorted(_a  ,key=lambda __UpperCAmelCase				: x["score"]  ,reverse=_a				)]
      # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
      if set(_a				).issubset(_a				):
          lowerCAmelCase__			:  Dict											= fill_masker(F"""This is a {tokenizer.mask_token}"""  ,top_k=3  ,targets=_a				)
          # They should yield exactly the same result
          self.assertEqual(nested_simplify(_a				)  ,nested_simplify(_a				)				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		Optional[Any]:
      lowerCAmelCase__			:  Optional[Any]											= FillMaskPipeline(model=_a  ,tokenizer=_a				)
      lowerCAmelCase__			:  Optional[Any]											= tokenizer.get_vocab()
      # String duplicates + id duplicates
      lowerCAmelCase__			:  str											= sorted(vocab.keys()				)[:3]
      lowerCAmelCase__			:  Tuple											= [targets[0], targets[1], targets[0], targets[2], targets[1]]
      lowerCAmelCase__			:  Tuple											= fill_masker(F"""My name is {tokenizer.mask_token}"""  ,targets=_a  ,top_k=10				)

      # The target list contains duplicates, so we can't output more
      # than them
      self.assertEqual(len(_a				)  ,3				)






  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		str:
      lowerCAmelCase__			:  str											= FillMaskPipeline(model=_a  ,tokenizer=_a				)

      lowerCAmelCase__			:  str											= fill_masker(
          F"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""  ,top_k=2				)
      self.assertEqual(
          _a  ,[
              [
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              ],
              [
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              ],
              [
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
                  {"""sequence""": ANY(_a				), """score""": ANY(_a				), """token""": ANY(_a				), """token_str""": ANY(_a				)},
              ],
          ]  ,)







'''simple docstring'''





import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase										=     logging.get_logger(__name__)

_lowerCAmelCase										=     {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_lowerCAmelCase										=     {
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

_lowerCAmelCase										=     {
    '''ctrl''': 256,
}

_lowerCAmelCase										=     {
    '''Pregnancy''': 16_8629,
    '''Christianity''': 7675,
    '''Explain''': 10_6423,
    '''Fitness''': 6_3440,
    '''Saving''': 6_3163,
    '''Ask''': 2_7171,
    '''Ass''': 9_5985,
    '''Joke''': 16_3509,
    '''Questions''': 4_5622,
    '''Thoughts''': 4_9605,
    '''Retail''': 5_2342,
    '''Feminism''': 16_4338,
    '''Writing''': 1_1992,
    '''Atheism''': 19_2263,
    '''Netflix''': 4_8616,
    '''Computing''': 3_9639,
    '''Opinion''': 4_3213,
    '''Alone''': 4_4967,
    '''Funny''': 5_8917,
    '''Gaming''': 4_0358,
    '''Human''': 4088,
    '''India''': 1331,
    '''Joker''': 7_7138,
    '''Diet''': 3_6206,
    '''Legal''': 1_1859,
    '''Norman''': 4939,
    '''Tip''': 7_2689,
    '''Weight''': 5_2343,
    '''Movies''': 4_6273,
    '''Running''': 2_3425,
    '''Science''': 2090,
    '''Horror''': 3_7793,
    '''Confession''': 6_0572,
    '''Finance''': 1_2250,
    '''Politics''': 1_6360,
    '''Scary''': 19_1985,
    '''Support''': 1_2654,
    '''Technologies''': 3_2516,
    '''Teenage''': 6_6160,
    '''Event''': 3_2769,
    '''Learned''': 6_7460,
    '''Notion''': 18_2770,
    '''Wikipedia''': 3_7583,
    '''Books''': 6665,
    '''Extract''': 7_6050,
    '''Confessions''': 10_2701,
    '''Conspiracy''': 7_5932,
    '''Links''': 6_3674,
    '''Narcissus''': 15_0425,
    '''Relationship''': 5_4766,
    '''Relationships''': 13_4796,
    '''Reviews''': 4_1671,
    '''News''': 4256,
    '''Translation''': 2_6820,
    '''multilingual''': 12_8406,
}
def        _SCREAMING_SNAKE_CASE  (	UpperCamelCase      ):



    """simple docstring"""





    lowerCAmelCase__			:  Dict											= set()
    lowerCAmelCase__			:  List[str]											= word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)      )
        lowerCAmelCase__			:  str											= char

    lowerCAmelCase__			:  int											= set(UpperCamelCase      )
    return pairs






class 	lowerCAmelCase_(    SCREAMING_SNAKE_CASE_					):
  '''simple docstring'''




  __lowercase							:				Optional[int]           =				VOCAB_FILES_NAMES
  __lowercase							:				Union[str, Any]           =				PRETRAINED_VOCAB_FILES_MAP
  __lowercase							:				List[Any]           =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowercase							:				str           =				CONTROL_CODES




  def __init__(   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase="<unk>"  ,**__UpperCAmelCase				)   ->		Optional[Any]:
      super().__init__(unk_token=__UpperCAmelCase  ,**__UpperCAmelCase				)

      with open(__UpperCAmelCase  ,encoding="""utf-8"""				) as vocab_handle:
          lowerCAmelCase__			:  List[Any]											= json.load(__UpperCAmelCase				)
      lowerCAmelCase__			:  str											= {v: k for k, v in self.encoder.items()}
      with open(__UpperCAmelCase  ,encoding="""utf-8"""				) as merges_handle:
          lowerCAmelCase__			:  Any											= merges_handle.read().split("""\n"""				)[1:-1]
      lowerCAmelCase__			:  Optional[Any]											= [tuple(merge.split()				) for merge in merges]
      lowerCAmelCase__			:  Tuple											= dict(zip(__UpperCAmelCase  ,range(len(__UpperCAmelCase				)				)				)				)
      lowerCAmelCase__			:  int											= {}




  @property
  def 							UpperCAmelCase_     (   self				)   ->		List[Any]:
      return len(self.encoder				)




  def 							UpperCAmelCase_     (   self				)   ->		Optional[int]:
      return dict(self.encoder  ,**self.added_tokens_encoder				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Optional[Any]:
      if token in self.cache:
          return self.cache[token]
      lowerCAmelCase__			:  int											= tuple(__UpperCAmelCase				)
      lowerCAmelCase__			:  str											= tuple(list(word[:-1]				) + [word[-1] + """</w>"""]				)
      lowerCAmelCase__			:  Tuple											= get_pairs(__UpperCAmelCase				)

      if not pairs:
          return token

      while True:
          lowerCAmelCase__			:  Tuple											= min(__UpperCAmelCase  ,key=lambda __UpperCAmelCase				: self.bpe_ranks.get(__UpperCAmelCase  ,float("""inf"""				)				)				)
          if bigram not in self.bpe_ranks:
              break
          lowerCAmelCase__				,       lowerCAmelCase__			:  Optional[Any]											= bigram
          lowerCAmelCase__			:  Dict											= []
          lowerCAmelCase__			:  int											= 0
          while i < len(__UpperCAmelCase				):
              try:
                  lowerCAmelCase__			:  Any											= word.index(__UpperCAmelCase  ,__UpperCAmelCase				)
              except ValueError:
                  new_word.extend(word[i:]				)
                  break
              else:
                  new_word.extend(word[i:j]				)
                  lowerCAmelCase__			:  Any											= j

              if word[i] == first and i < len(__UpperCAmelCase				) - 1 and word[i + 1] == second:
                  new_word.append(first + second				)
                  i += 2
              else:
                  new_word.append(word[i]				)
                  i += 1
          lowerCAmelCase__			:  Any											= tuple(__UpperCAmelCase				)
          lowerCAmelCase__			:  Tuple											= new_word
          if len(__UpperCAmelCase				) == 1:
              break
          else:
              lowerCAmelCase__			:  List[str]											= get_pairs(__UpperCAmelCase				)
      lowerCAmelCase__			:  List[Any]											= """@@ """.join(__UpperCAmelCase				)
      lowerCAmelCase__			:  int											= word[:-4]
      lowerCAmelCase__			:  Optional[Any]											= word
      return word




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Union[str, Any]:
      lowerCAmelCase__			:  List[Any]											= []

      lowerCAmelCase__			:  Optional[int]											= re.findall(R"""\S+\n?"""  ,__UpperCAmelCase				)

      for token in words:
          split_tokens.extend(list(self.bpe(__UpperCAmelCase				).split(""" """				)				)				)
      return split_tokens




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Tuple:
      return self.encoder.get(__UpperCAmelCase  ,self.encoder.get(self.unk_token				)				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Optional[int]:
      return self.decoder.get(__UpperCAmelCase  ,self.unk_token				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase				)   ->		Dict:
      lowerCAmelCase__			:  Optional[int]											= """ """.join(__UpperCAmelCase				).replace("""@@ """  ,""""""				).strip()
      return out_string






  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase = None				)   ->		Tuple[str]:
      if not os.path.isdir(__UpperCAmelCase				):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
          return
      lowerCAmelCase__			:  Tuple											= os.path.join(
          __UpperCAmelCase  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
      lowerCAmelCase__			:  str											= os.path.join(
          __UpperCAmelCase  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]				)

      with open(__UpperCAmelCase  ,"""w"""  ,encoding="""utf-8"""				) as f:
          f.write(json.dumps(self.encoder  ,indent=2  ,sort_keys=__UpperCAmelCase  ,ensure_ascii=__UpperCAmelCase				) + """\n"""				)

      lowerCAmelCase__			:  Optional[int]											= 0
      with open(__UpperCAmelCase  ,"""w"""  ,encoding="""utf-8"""				) as writer:
          writer.write("""#version: 0.2\n"""				)
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,key=lambda __UpperCAmelCase				: kv[1]				):
              if index != token_index:
                  logger.warning(
                      F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                      """ Please check that the tokenizer is not corrupted!"""				)
                  lowerCAmelCase__			:  List[str]											= token_index
              writer.write(""" """.join(__UpperCAmelCase				) + """\n"""				)
              index += 1

      return vocab_file, merge_file

  # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
  #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
  #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
  #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
  #     return ''.join(tokens_generated_so_far)


