





import unittest

from transformers import BertGenerationTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case           =			"""▁"""

_snake_case           =			get_tests_dir("""fixtures/test_sentencepiece.model""")





@require_sentencepiece
class  lowerCAmelCase							(       lowercase_							,							unittest.TestCase ):
  __lowerCamelCase		=      BertGenerationTokenizer
  __lowerCamelCase		=      False
  __lowerCamelCase		=      True







  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     super().setUp()

     lowercase__			       =  BertGenerationTokenizer(_lowercase   ,							keep_accents=_lowercase )
     tokenizer.save_pretrained(self.tmpdirname )







  def   UpperCAmelCase					(      self  :Dict ):






     '''simple docstring'''





     lowercase__			       =  "<s>"
     lowercase__			       =  1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowercase )   ,							_lowercase )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowercase )   ,							_lowercase )







  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     lowercase__			       =  list(self.get_tokenizer().get_vocab().keys() )

     self.assertEqual(vocab_keys[0]   ,							"<unk>" )
     self.assertEqual(vocab_keys[1]   ,							"<s>" )
     self.assertEqual(vocab_keys[-1]   ,							"<pad>" )
     self.assertEqual(len(_lowercase )   ,							10_02 )







  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     self.assertEqual(self.get_tokenizer().vocab_size   ,							10_00 )







  def   UpperCAmelCase					(      self  :Union[str, Any] ):






     '''simple docstring'''





     lowercase__			       =  BertGenerationTokenizer(_lowercase   ,							keep_accents=_lowercase )

     lowercase__			       =  tokenizer.tokenize("This is a test" )
     self.assertListEqual(_lowercase   ,							["▁This", "▁is", "▁a", "▁t", "est"] )

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(_lowercase )   ,							[2_85, 46, 10, 1_70, 3_82]   ,							)

     lowercase__			       =  tokenizer.tokenize("I was born in 92000, and this is falsé." )
     self.assertListEqual(
         _lowercase   ,							[
             SPIECE_UNDERLINE + "I",
             SPIECE_UNDERLINE + "was",
             SPIECE_UNDERLINE + "b",
             "or",
             "n",
             SPIECE_UNDERLINE + "in",
             SPIECE_UNDERLINE + "",
             "9",
             "2",
             "0",
             "0",
             "0",
             ",",
             SPIECE_UNDERLINE + "and",
             SPIECE_UNDERLINE + "this",
             SPIECE_UNDERLINE + "is",
             SPIECE_UNDERLINE + "f",
             "al",
             "s",
             "é",
             ".",
         ]   ,							)
     lowercase__			       =  tokenizer.convert_tokens_to_ids(_lowercase )
     self.assertListEqual(
         _lowercase   ,							[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]   ,							)

     lowercase__			       =  tokenizer.convert_ids_to_tokens(_lowercase )
     self.assertListEqual(
         _lowercase   ,							[
             SPIECE_UNDERLINE + "I",
             SPIECE_UNDERLINE + "was",
             SPIECE_UNDERLINE + "b",
             "or",
             "n",
             SPIECE_UNDERLINE + "in",
             SPIECE_UNDERLINE + "",
             "<unk>",
             "2",
             "0",
             "0",
             "0",
             ",",
             SPIECE_UNDERLINE + "and",
             SPIECE_UNDERLINE + "this",
             SPIECE_UNDERLINE + "is",
             SPIECE_UNDERLINE + "f",
             "al",
             "s",
             "<unk>",
             ".",
         ]   ,							)







  @cached_property
  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     return BertGenerationTokenizer.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder" )







  @slow
  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     lowercase__			       =  "Hello World!"
     lowercase__			       =  [1_85_36, 22_60, 1_01]

     self.assertListEqual(_lowercase   ,							self.big_tokenizer.encode(_lowercase ) )







  @slow
  def   UpperCAmelCase					(      self  :Dict ):






     '''simple docstring'''





     lowercase__			       =  (
         "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
         " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
     )
     lowercase__			       =  [
         8_71,
         4_19,
         3_58,
         9_46,
         9_91,
         25_21,
         4_52,
         3_58,
         13_57,
         3_87,
         77_51,
         35_36,
         1_12,
         9_85,
         4_56,
         1_26,
         8_65,
         9_38,
         54_00,
         57_34,
         4_58,
         13_68,
         4_67,
         7_86,
         24_62,
         52_46,
         11_59,
         6_33,
         8_65,
         45_19,
         4_57,
         5_82,
         8_52,
         25_57,
         4_27,
         9_16,
         5_08,
         4_05,
         3_43_24,
         4_97,
         3_91,
         4_08,
         1_13_42,
         12_44,
         3_85,
         1_00,
         9_38,
         9_85,
         4_56,
         5_74,
         3_62,
         1_25_97,
         32_00,
         31_29,
         11_72,
     ]

     self.assertListEqual(_lowercase   ,							self.big_tokenizer.encode(_lowercase ) )







  @require_torch
  @slow
  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     import torch

     from transformers import BertGenerationConfig, BertGenerationEncoder

     # Build sequence
     lowercase__			       =  list(self.big_tokenizer.get_vocab().keys() )[:10]
     lowercase__			       =  " ".join(_lowercase )
     lowercase__			       =  self.big_tokenizer.encode_plus(_lowercase   ,							return_tensors="pt"   ,							return_token_type_ids=_lowercase )
     lowercase__			       =  self.big_tokenizer.batch_encode_plus(
         [sequence + " " + sequence]   ,							return_tensors="pt"   ,							return_token_type_ids=_lowercase )

     lowercase__			       =  BertGenerationConfig()
     lowercase__			       =  BertGenerationEncoder(_lowercase )

     assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

     with torch.no_grad():
        model(**_lowercase )
        model(**_lowercase )







  @slow
  def   UpperCAmelCase					(      self  :List[str] ):






     '''simple docstring'''





     lowercase__			       =  {"input_ids": [[3_92_86, 4_58, 3_63_35, 20_01, 4_56, 1_30_73, 1_32_66, 4_55, 1_13, 77_46, 17_41, 1_11_57, 3_91, 1_30_73, 1_32_66, 4_55, 1_13, 39_67, 3_54_12, 1_13, 49_36, 1_09, 38_70, 23_77, 1_13, 3_00_84, 4_57_20, 4_58, 1_34, 1_74_96, 1_12, 5_03, 1_16_72, 1_13, 1_18, 1_12, 56_65, 1_33_47, 3_86_87, 1_12, 14_96, 3_13_89, 1_12, 32_68, 4_72_64, 1_34, 9_62, 1_12, 1_63_77, 80_35, 2_31_30, 4_30, 1_21_69, 1_55_18, 2_85_92, 4_58, 1_46, 4_16_97, 1_09, 3_91, 1_21_69, 1_55_18, 1_66_89, 4_58, 1_46, 4_13_58, 1_09, 4_52, 7_26, 40_34, 1_11, 7_63, 3_54_12, 50_82, 3_88, 19_03, 1_11, 90_51, 3_91, 28_70, 4_89_18, 19_00, 11_23, 5_50, 9_98, 1_12, 95_86, 1_59_85, 4_55, 3_91, 4_10, 2_29_55, 3_76_36, 1_14], [4_48, 1_74_96, 4_19, 36_63, 3_85, 7_63, 1_13, 2_75_33, 28_70, 32_83, 1_30_43, 16_39, 2_47_13, 5_23, 6_56, 2_40_13, 1_85_50, 25_21, 5_17, 2_70_14, 2_12_44, 4_20, 12_12, 14_65, 3_91, 9_27, 48_33, 3_88, 5_78, 1_17_86, 1_14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4_84, 21_69, 76_87, 2_19_32, 1_81_46, 7_26, 3_63, 1_70_32, 33_91, 1_14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=_lowercase   ,							model_name="google/bert_for_seq_generation_L-24_bbc_encoder"   ,							revision="c817d1fd1be2ffa69431227a1fe320544943d4db"   ,							)













from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml





class  lowerCAmelCase							(       yaml.SafeLoader ):







  def   UpperCAmelCase					(      self  :Optional[Any]   ,							_lowercase  :Any ):






     '''simple docstring'''





     lowercase__			       =  [self.constructed_objects[key_node] for key_node, _ in node.value]
     lowercase__			       =  [tuple(_lowercase ) if isinstance(_lowercase   ,							_lowercase ) else key for key in keys]
     lowercase__			       =  Counter(_lowercase )
     lowercase__			       =  [key for key in counter if counter[key] > 1]
     if duplicate_keys:
        raise TypeError(f'''Got duplicate yaml keys: {duplicate_keys}''' )







  def   UpperCAmelCase					(      self  :Any   ,							_lowercase  :str   ,							_lowercase  :Dict=False ):






     '''simple docstring'''





     lowercase__			       =  super().construct_mapping(_lowercase   ,							deep=_lowercase )
     self._check_no_duplicates_on_constructed_node(_lowercase )
     return mapping


def _A       (   __magic_name__	):
   lowercase__			       =  list(readme_content.splitlines()	)
   if full_content and full_content[0] == "---" and "---" in full_content[1:]:
      lowercase__			       =  full_content[1:].index("---"	) + 1
      lowercase__			       =  "\n".join(full_content[1:sep_idx]	)
      return yamlblock, "\n".join(full_content[sep_idx + 1 :]	)

   return None, "\n".join(__magic_name__	)





class  lowerCAmelCase							(       lowercase_ ):
  # class attributes
  __lowerCamelCase		=      {'train_eval_index'}  # train-eval-index in the YAML metadata







  @classmethod
  def   UpperCAmelCase					(      cls  :Dict   ,							_lowercase  :Path ):






     '''simple docstring'''





     with open(_lowercase   ,							encoding="utf-8" ) as readme_file:
        lowercase__       ,							lowercase__			       =  _split_yaml_from_readme(readme_file.read() )
     if yaml_string is not None:
        return cls.from_yaml_string(_lowercase )
     else:
        return cls()







  def   UpperCAmelCase					(      self  :Any   ,							_lowercase  :Path ):






     '''simple docstring'''





     if path.exists():
        with open(_lowercase   ,							encoding="utf-8" ) as readme_file:
           lowercase__			       =  readme_file.read()
     else:
        lowercase__			       =  None
     lowercase__			       =  self._to_readme(_lowercase )
     with open(_lowercase   ,							"w"   ,							encoding="utf-8" ) as readme_file:
        readme_file.write(_lowercase )







  def   UpperCAmelCase					(      self  :List[str]   ,							_lowercase  :Optional[str] = None ):






     '''simple docstring'''





     if readme_content is not None:
        lowercase__       ,							lowercase__			       =  _split_yaml_from_readme(_lowercase )
        lowercase__			       =  "---\n" + self.to_yaml_string() + "---\n" + content
     else:
        lowercase__			       =  "---\n" + self.to_yaml_string() + "---\n"
     return full_content







  @classmethod
  def   UpperCAmelCase					(      cls  :Any   ,							_lowercase  :str ):






     '''simple docstring'''





     lowercase__			       =  yaml.load(_lowercase   ,							Loader=_NoDuplicateSafeLoader ) or {}

     # Convert the YAML keys to DatasetMetadata fields
     lowercase__			       =  {
         (key.replace("-"   ,							"_" ) if key.replace("-"   ,							"_" ) in cls._FIELDS_WITH_DASHES else key): value
         for key, value in metadata_dict.items()
     }
     return cls(**_lowercase )







  def   UpperCAmelCase					(      self  :Union[str, Any] ):






     '''simple docstring'''





     return yaml.safe_dump(
         {
             (key.replace("_"   ,							"-" ) if key in self._FIELDS_WITH_DASHES else key): value
             for key, value in self.items()
         }   ,							sort_keys=_lowercase   ,							allow_unicode=_lowercase   ,							encoding="utf-8"   ,							).decode("utf-8" )
_snake_case           =			{
    """image-classification""": [],
    """translation""": [],
    """image-segmentation""": [],
    """fill-mask""": [],
    """automatic-speech-recognition""": [],
    """token-classification""": [],
    """sentence-similarity""": [],
    """audio-classification""": [],
    """question-answering""": [],
    """summarization""": [],
    """zero-shot-classification""": [],
    """table-to-text""": [],
    """feature-extraction""": [],
    """other""": [],
    """multiple-choice""": [],
    """text-classification""": [],
    """text-to-image""": [],
    """text2text-generation""": [],
    """zero-shot-image-classification""": [],
    """tabular-classification""": [],
    """tabular-regression""": [],
    """image-to-image""": [],
    """tabular-to-text""": [],
    """unconditional-image-generation""": [],
    """text-retrieval""": [],
    """text-to-speech""": [],
    """object-detection""": [],
    """audio-to-audio""": [],
    """text-generation""": [],
    """conversational""": [],
    """table-question-answering""": [],
    """visual-question-answering""": [],
    """image-to-text""": [],
    """reinforcement-learning""": [],
    """voice-activity-detection""": [],
    """time-series-forecasting""": [],
    """document-question-answering""": [],
}


if __name__ == "__main__":
  from argparse import ArgumentParser

  _snake_case           =			ArgumentParser(usage="""Validate the yaml metadata block of a README.md file.""")
  ap.add_argument("""readme_filepath""")
  _snake_case           =			ap.parse_args()

  _snake_case           =			Path(args.readme_filepath)
  _snake_case           =			DatasetMetadata.from_readme(readme_filepath)
  print(dataset_metadata)
  dataset_metadata.to_readme(readme_filepath)






