



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class      UpperCAmelCase_  (    _lowerCamelCase	,      unittest.TestCase						):
      __SCREAMING_SNAKE_CASE						:       List[str]								=			BertJapaneseTokenizer
      __SCREAMING_SNAKE_CASE						:       Union[str, Any]								=			False
      __SCREAMING_SNAKE_CASE						:       Union[str, Any]								=			True



      def 						snake_case_ (					self					:     List[Any]						):
           super().setUp()

           _UpperCAmelCase      :     Optional[Any]        =   [
               "[UNK]",
               "[CLS]",
               "[SEP]",
               "こんにちは",
               "こん",
               "にちは",
               "ばんは",
               "##こん",
               "##にちは",
               "##ばんは",
               "世界",
               "##世界",
               "、",
               "##、",
               "。",
               "##。",
           ]

           _UpperCAmelCase      :     Tuple        =   os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES["vocab_file"]						)
           with open(self.vocab_file					,     "w"					,     encoding="utf-8"						) as vocab_writer:
                vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)



      def 						snake_case_ (					self					:     str					,     A					:     List[Any]						):
           _UpperCAmelCase      :     Tuple        =   "こんにちは、世界。 \nこんばんは、世界。"
           _UpperCAmelCase      :     Any        =   "こんにちは 、 世界 。 こんばんは 、 世界 。"
           return input_text, output_text



      def 						snake_case_ (					self					:     Any					,     A					:     Tuple						):
           _UpperCAmelCase    ,	_UpperCAmelCase      :     str        =   self.get_input_output_texts(A						)
           _UpperCAmelCase      :     List[str]        =   tokenizer.encode(A					,     add_special_tokens=A						)
           _UpperCAmelCase      :     str        =   tokenizer.decode(A					,     clean_up_tokenization_spaces=A						)
           return text, ids



      def 						snake_case_ (					self					:     Optional[int]						):
           pass  # TODO add if relevant



      def 						snake_case_ (					self					:     List[Any]						):
           pass  # TODO add if relevant



      def 						snake_case_ (					self					:     Dict						):
           pass  # TODO add if relevant



      def 						snake_case_ (					self					:     List[Any]						):
           _UpperCAmelCase      :     List[str]        =   self.tokenizer_class(self.vocab_file						)

           _UpperCAmelCase      :     str        =   tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"						)
           self.assertListEqual(A					,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(A						)					,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]						)



      def 						snake_case_ (					self					:     Optional[int]						):
           _UpperCAmelCase      :     List[Any]        =   self.tokenizer_class(self.vocab_file					,     word_tokenizer_type="mecab"						)
           self.assertIsNotNone(A						)

           _UpperCAmelCase      :     Tuple        =   "こんにちは、世界。\nこんばんは、世界。"
           _UpperCAmelCase      :     Union[str, Any]        =   tokenizer.tokenize(A						)
           self.assertListEqual(A					,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(A						)					,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]						)

           _UpperCAmelCase      :     Optional[Any]        =   os.path.join(self.tmpdirname					,     "tokenizer.bin"						)
           with open(A					,     "wb"						) as handle:
                pickle.dump(A					,     A						)

           with open(A					,     "rb"						) as handle:
                _UpperCAmelCase      :     str        =   pickle.load(A						)

           _UpperCAmelCase      :     Union[str, Any]        =   tokenizer_new.tokenize(A						)

           self.assertListEqual(A					,     A						)



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     List[Any]        =   MecabTokenizer(mecab_dic="ipadic"						)

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					,     )



      def 						snake_case_ (					self					:     Dict						):


           try:
                _UpperCAmelCase      :     Any        =   MecabTokenizer(mecab_dic="unidic_lite"						)
           except ModuleNotFoundError:
                return

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					,     )



      def 						snake_case_ (					self					:     Tuple						):


           try:
                _UpperCAmelCase      :     Any        =   MecabTokenizer(mecab_dic="unidic"						)
           except ModuleNotFoundError:
                return

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					,     )



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     Tuple        =   MecabTokenizer(do_lower_case=A					,     mecab_dic="ipadic"						)

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]					,     )



      def 						snake_case_ (					self					:     List[str]						):


           try:
                _UpperCAmelCase      :     Optional[Any]        =   MecabTokenizer(
                    do_lower_case=A					,     normalize_text=A					,     mecab_option="-d /usr/local/lib/mecab/dic/jumandic"						)
           except RuntimeError:
                # if dict doesn't exist in the system, previous code raises this error.
                return

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]					,     )



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     int        =   MecabTokenizer(normalize_text=A					,     mecab_dic="ipadic"						)

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]					,     )



      @require_sudachi
      def 						snake_case_ (					self					:     List[Any]						):
           _UpperCAmelCase      :     Optional[int]        =   self.tokenizer_class(self.vocab_file					,     word_tokenizer_type="sudachi"						)
           self.assertIsNotNone(A						)

           _UpperCAmelCase      :     Union[str, Any]        =   "こんにちは、世界。\nこんばんは、世界。"
           _UpperCAmelCase      :     Any        =   tokenizer.tokenize(A						)
           self.assertListEqual(A					,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(A						)					,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]						)

           _UpperCAmelCase      :     Dict        =   os.path.join(self.tmpdirname					,     "tokenizer.bin"						)
           with open(A					,     "wb"						) as handle:
                pickle.dump(A					,     A						)

           with open(A					,     "rb"						) as handle:
                _UpperCAmelCase      :     Dict        =   pickle.load(A						)

           _UpperCAmelCase      :     Tuple        =   tokenizer_new.tokenize(A						)

           self.assertListEqual(A					,     A						)



      @require_sudachi
      def 						snake_case_ (					self					:     Any						):
           _UpperCAmelCase      :     Any        =   SudachiTokenizer(sudachi_dict_type="core"						)

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]					,     )



      @require_sudachi
      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     Dict        =   SudachiTokenizer(sudachi_dict_type="core"					,     sudachi_split_mode="A"						)

           self.assertListEqual(tokenizer.tokenize("外国人参政権"						)					,     ["外国", "人", "参政", "権"]						)



      @require_sudachi
      def 						snake_case_ (					self					:     Optional[int]						):
           _UpperCAmelCase      :     str        =   SudachiTokenizer(sudachi_dict_type="core"					,     sudachi_split_mode="B"						)

           self.assertListEqual(tokenizer.tokenize("外国人参政権"						)					,     ["外国人", "参政権"]						)



      @require_sudachi
      def 						snake_case_ (					self					:     Any						):
           _UpperCAmelCase      :     List[Any]        =   SudachiTokenizer(sudachi_dict_type="core"					,     sudachi_split_mode="C"						)

           self.assertListEqual(tokenizer.tokenize("外国人参政権"						)					,     ["外国人参政権"]						)



      @require_sudachi
      def 						snake_case_ (					self					:     Optional[int]						):
           _UpperCAmelCase      :     int        =   SudachiTokenizer(do_lower_case=A					,     sudachi_dict_type="core"						)

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]					,     )



      @require_sudachi
      def 						snake_case_ (					self					:     List[Any]						):
           _UpperCAmelCase      :     Dict        =   SudachiTokenizer(normalize_text=A					,     sudachi_dict_type="core"						)

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]					,     )



      @require_sudachi
      def 						snake_case_ (					self					:     List[str]						):
           _UpperCAmelCase      :     int        =   SudachiTokenizer(trim_whitespace=A					,     sudachi_dict_type="core"						)

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					,     )



      @require_jumanpp
      def 						snake_case_ (					self					:     int						):
           _UpperCAmelCase      :     List[str]        =   self.tokenizer_class(self.vocab_file					,     word_tokenizer_type="jumanpp"						)
           self.assertIsNotNone(A						)

           _UpperCAmelCase      :     Optional[Any]        =   "こんにちは、世界。\nこんばんは、世界。"
           _UpperCAmelCase      :     Optional[Any]        =   tokenizer.tokenize(A						)
           self.assertListEqual(A					,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(A						)					,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]						)

           _UpperCAmelCase      :     Any        =   os.path.join(self.tmpdirname					,     "tokenizer.bin"						)
           with open(A					,     "wb"						) as handle:
                pickle.dump(A					,     A						)

           with open(A					,     "rb"						) as handle:
                _UpperCAmelCase      :     Optional[Any]        =   pickle.load(A						)

           _UpperCAmelCase      :     Any        =   tokenizer_new.tokenize(A						)

           self.assertListEqual(A					,     A						)



      @require_jumanpp
      def 						snake_case_ (					self					:     Any						):
           _UpperCAmelCase      :     str        =   JumanppTokenizer()

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]					,     )



      @require_jumanpp
      def 						snake_case_ (					self					:     List[Any]						):
           _UpperCAmelCase      :     str        =   JumanppTokenizer(do_lower_case=A						)

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]					,     )



      @require_jumanpp
      def 						snake_case_ (					self					:     Optional[Any]						):
           _UpperCAmelCase      :     Dict        =   JumanppTokenizer(normalize_text=A						)

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]					,     )



      @require_jumanpp
      def 						snake_case_ (					self					:     List[Any]						):
           _UpperCAmelCase      :     List[Any]        =   JumanppTokenizer(trim_whitespace=A						)

           self.assertListEqual(
               tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)					,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]					,     )



      @require_jumanpp
      def 						snake_case_ (					self					:     Optional[Any]						):
           _UpperCAmelCase      :     Union[str, Any]        =   JumanppTokenizer()

           self.assertListEqual(
               tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"						)					,     ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]					,     )



      def 						snake_case_ (					self					:     int						):
           _UpperCAmelCase      :     str        =   ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

           _UpperCAmelCase      :     List[str]        =   {}
           for i, token in enumerate(A						):
                _UpperCAmelCase      :     List[Any]        =   i
           _UpperCAmelCase      :     Optional[Any]        =   WordpieceTokenizer(vocab=A					,     unk_token="[UNK]"						)

           self.assertListEqual(tokenizer.tokenize(""						)					,     []						)

           self.assertListEqual(tokenizer.tokenize("こんにちは"						)					,     ["こんにちは"]						)

           self.assertListEqual(tokenizer.tokenize("こんばんは"						)					,     ["こん", "##ばんは"]						)

           self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"						)					,     ["こん", "##ばんは", "[UNK]", "こんにちは"]						)



      def 						snake_case_ (					self					:     Union[str, Any]						):
           _UpperCAmelCase      :     Optional[int]        =   BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"						)
           _UpperCAmelCase      :     Any        =   tokenizer.subword_tokenizer

           _UpperCAmelCase      :     Union[str, Any]        =   subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"						)
           self.assertListEqual(A					,     ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]						)

           _UpperCAmelCase      :     Optional[int]        =   subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"						)
           self.assertListEqual(A					,     ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]						)



      def 						snake_case_ (					self					:     str						):
           _UpperCAmelCase      :     Tuple        =   self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"						)

           _UpperCAmelCase      :     Any        =   tokenizer.encode("ありがとう。"					,     add_special_tokens=A						)
           _UpperCAmelCase      :     int        =   tokenizer.encode("どういたしまして。"					,     add_special_tokens=A						)

           _UpperCAmelCase      :     Tuple        =   tokenizer.build_inputs_with_special_tokens(A						)
           _UpperCAmelCase      :     Tuple        =   tokenizer.build_inputs_with_special_tokens(A					,     A						)

           # 2 is for "[CLS]", 3 is for "[SEP]"
           assert encoded_sentence == [2] + text + [3]
           assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class      UpperCAmelCase_  (    _lowerCamelCase	,      unittest.TestCase						):
      __SCREAMING_SNAKE_CASE						:       Any								=			BertJapaneseTokenizer
      __SCREAMING_SNAKE_CASE						:       Any								=			False



      def 						snake_case_ (					self					:     Union[str, Any]						):
           super().setUp()

           _UpperCAmelCase      :     str        =   ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

           _UpperCAmelCase      :     Optional[Any]        =   os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES["vocab_file"]						)
           with open(self.vocab_file					,     "w"					,     encoding="utf-8"						) as vocab_writer:
                vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)



      def 						snake_case_ (					self					:     str					,     **A					:     List[str]						):
           return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,     subword_tokenizer_type="character"					,     **A						)



      def 						snake_case_ (					self					:     Dict					,     A					:     Tuple						):
           _UpperCAmelCase      :     List[Any]        =   "こんにちは、世界。 \nこんばんは、世界。"
           _UpperCAmelCase      :     List[Any]        =   "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
           return input_text, output_text



      def 						snake_case_ (					self					:     int						):
           pass  # TODO add if relevant



      def 						snake_case_ (					self					:     List[str]						):
           pass  # TODO add if relevant



      def 						snake_case_ (					self					:     int						):
           pass  # TODO add if relevant



      def 						snake_case_ (					self					:     Union[str, Any]						):
           _UpperCAmelCase      :     Union[str, Any]        =   self.tokenizer_class(self.vocab_file					,     subword_tokenizer_type="character"						)

           _UpperCAmelCase      :     str        =   tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"						)
           self.assertListEqual(
               A					,     ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]						)
           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(A						)					,     [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]						)



      def 						snake_case_ (					self					:     Tuple						):
           _UpperCAmelCase      :     Tuple        =   ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

           _UpperCAmelCase      :     int        =   {}
           for i, token in enumerate(A						):
                _UpperCAmelCase      :     List[Any]        =   i
           _UpperCAmelCase      :     Optional[int]        =   CharacterTokenizer(vocab=A					,     unk_token="[UNK]"						)

           self.assertListEqual(tokenizer.tokenize(""						)					,     []						)

           self.assertListEqual(tokenizer.tokenize("こんにちは"						)					,     ["こ", "ん", "に", "ち", "は"]						)

           self.assertListEqual(tokenizer.tokenize("こんにちほ"						)					,     ["こ", "ん", "に", "ち", "[UNK]"]						)



      def 						snake_case_ (					self					:     List[str]						):
           _UpperCAmelCase      :     str        =   self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"						)

           _UpperCAmelCase      :     Optional[int]        =   tokenizer.encode("ありがとう。"					,     add_special_tokens=A						)
           _UpperCAmelCase      :     Union[str, Any]        =   tokenizer.encode("どういたしまして。"					,     add_special_tokens=A						)

           _UpperCAmelCase      :     Tuple        =   tokenizer.build_inputs_with_special_tokens(A						)
           _UpperCAmelCase      :     Any        =   tokenizer.build_inputs_with_special_tokens(A					,     A						)

           # 2 is for "[CLS]", 3 is for "[SEP]"
           assert encoded_sentence == [2] + text + [3]
           assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class      UpperCAmelCase_  (    unittest.TestCase						):



      def 						snake_case_ (					self					:     str						):
           _UpperCAmelCase      :     Dict        =   "cl-tohoku/bert-base-japanese"
           _UpperCAmelCase      :     List[str]        =   AutoTokenizer.from_pretrained(A						)
           self.assertIsInstance(A					,     A						)







class      UpperCAmelCase_  (    unittest.TestCase						):



      def 						snake_case_ (					self					:     Union[str, Any]						):
           _UpperCAmelCase      :     List[str]        =   "cl-tohoku/bert-base-japanese"
           with self.assertLogs("transformers"					,     level="WARNING"						) as cm:
                BertTokenizer.from_pretrained(A						)
                self.assertTrue(
                    cm.records[0].message.startswith(
                        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                        " is called from."						)						)
           _UpperCAmelCase      :     int        =   "bert-base-cased"
           with self.assertLogs("transformers"					,     level="WARNING"						) as cm:
                BertJapaneseTokenizer.from_pretrained(A						)
                self.assertTrue(
                    cm.records[0].message.startswith(
                        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                        " is called from."						)						)





"""simple docstring"""




import importlib
import inspect
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
_lowerCAmelCase    : Dict							=							"src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
_lowerCAmelCase    : List[Any]							=							importlib.util.spec_from_file_location(
    "transformers",
    os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"),
    submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
_lowerCAmelCase    : Tuple							=							spec.loader.load_module()

_lowerCAmelCase    : Tuple							=							transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
_lowerCAmelCase    : Optional[int]							=							re.compile("\[(.+?)\]\((https://huggingface\.co/.+?)\)")


_lowerCAmelCase    : Optional[int]							=							{
    "CLIPConfigMixin",
    "DecisionTransformerConfigMixin",
    "EncoderDecoderConfigMixin",
    "RagConfigMixin",
    "SpeechEncoderDecoderConfigMixin",
    "VisionEncoderDecoderConfigMixin",
    "VisionTextDualEncoderConfigMixin",
}

def 						__snake_case						(				)  ->       Dict:





     '''simple docstring'''

     _UpperCAmelCase      :     List[str]        =   []

     for config_class in list(CONFIG_MAPPING.values()    ):
          _UpperCAmelCase      :     Union[str, Any]        =   False

          # source code of `config_class`
          _UpperCAmelCase      :     Optional[int]        =   inspect.getsource(SCREAMING_SNAKE_CASE__    )
          _UpperCAmelCase      :     List[Any]        =   _re_checkpoint.findall(SCREAMING_SNAKE_CASE__    )

          for checkpoint in checkpoints:
               # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
               # For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
               _UpperCAmelCase    ,	_UpperCAmelCase      :     List[Any]        =   checkpoint

               # verify the checkpoint name corresponds to the checkpoint link
               _UpperCAmelCase      :     Optional[Any]        =   f'https://huggingface.co/{ckpt_name}'
               if ckpt_link == ckpt_link_from_name:
                    _UpperCAmelCase      :     Optional[Any]        =   True
                    break

          _UpperCAmelCase      :     int        =   config_class.__name__
          if not checkpoint_found and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
               configs_without_checkpoint.append(SCREAMING_SNAKE_CASE__    )

     if len(SCREAMING_SNAKE_CASE__    ) > 0:
          _UpperCAmelCase      :     List[str]        =   "\n".join(sorted(SCREAMING_SNAKE_CASE__    )    )
          raise ValueError(f'The following configurations don\'t contain any valid checkpoint:\n{message}'    )


if __name__ == "__main__":
 check_config_docstrings_have_checkpoints()
