


'''simple docstring'''



import argparse
from collections import defaultdict

import yaml


__UpperCAmelCase      :int			    =       "docs/source/en/_toctree.yml"




def      _a     (						_lowercase :     List[str]  ):





     '''simple docstring'''






     __UpperCAmelCase     :			str  							=      defaultdict(_lowercase  )
     __UpperCAmelCase     :			str  							=      []
     __UpperCAmelCase     :			Optional[int]  							=      []
     for doc in doc_list:
          if "local" in doc:
               counts[doc["local"]] += 1

          if doc["title"].lower() == "overview":
               overview_doc.append({'''local''': doc['''local'''], '''title''': doc['''title''']}  )
          else:
               new_doc_list.append(_lowercase  )

     __UpperCAmelCase     :			List[str]  							=      new_doc_list
     __UpperCAmelCase     :			Tuple  							=      [key for key, value in counts.items() if value > 1]

     __UpperCAmelCase     :			Optional[Any]  							=      []
     for duplicate_key in duplicates:
          __UpperCAmelCase     :			Tuple  							=      list({doc['''title'''] for doc in doc_list if doc['''local'''] == duplicate_key}  )
          if len(_lowercase  ) > 1:
               raise ValueError(
                   F'{duplicate_key} is present several times in the documentation table of content at '
                   '''`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the '''
                   '''others.'''  )
          # Only add this once
          new_doc.append({'''local''': duplicate_key, '''title''': titles[0]}  )

     # Add none duplicate-keys
     new_doc.extend([doc for doc in doc_list if '''local''' not in counts or counts[doc['''local''']] == 1]  )
     __UpperCAmelCase     :			Optional[int]  							=      sorted(_lowercase      , key=lambda _lowercase  : s["title"].lower()  )

     # "overview" gets special treatment and is always first
     if len(_lowercase  ) > 1:
          raise ValueError('''{doc_list} has two \'overview\' docs which is not allowed.'''  )

     overview_doc.extend(_lowercase  )

     # Sort
     return overview_doc




def      _a     (						_lowercase :     List[str]=False  ):





     '''simple docstring'''






     with open(_lowercase      , encoding='''utf-8'''  ) as f:
          __UpperCAmelCase     :			Union[str, Any]  							=      yaml.safe_load(f.read()  )

     # Get to the API doc
     __UpperCAmelCase     :			Dict  							=      0
     while content[api_idx]["title"] != "API":
          api_idx += 1
     __UpperCAmelCase     :			List[Any]  							=      content[api_idx]['''sections''']

     # Then to the model doc
     __UpperCAmelCase     :			Tuple  							=      0
     while api_doc[scheduler_idx]["title"] != "Schedulers":
          scheduler_idx += 1

     __UpperCAmelCase     :			Dict  							=      api_doc[scheduler_idx]['''sections''']
     __UpperCAmelCase     :			List[Any]  							=      clean_doc_toc(_lowercase  )

     __UpperCAmelCase     :			Optional[Any]  							=      False
     if new_scheduler_doc != scheduler_doc:
          __UpperCAmelCase     :			Union[str, Any]  							=      True
          if overwrite:
               __UpperCAmelCase     :			Optional[Any]  							=      new_scheduler_doc

     if diff:
          if overwrite:
               __UpperCAmelCase     :			str  							=      api_doc
               with open(_lowercase      , '''w'''      , encoding='''utf-8'''  ) as f:
                    f.write(yaml.dump(_lowercase      , allow_unicode=_lowercase  )  )
          else:
               raise ValueError(
                   '''The model doc part of the table of content is not properly sorted, run `make style` to fix this.'''  )




def      _a     (						_lowercase :     Optional[Any]=False  ):





     '''simple docstring'''






     with open(_lowercase      , encoding='''utf-8'''  ) as f:
          __UpperCAmelCase     :			Union[str, Any]  							=      yaml.safe_load(f.read()  )

     # Get to the API doc
     __UpperCAmelCase     :			List[Any]  							=      0
     while content[api_idx]["title"] != "API":
          api_idx += 1
     __UpperCAmelCase     :			Union[str, Any]  							=      content[api_idx]['''sections''']

     # Then to the model doc
     __UpperCAmelCase     :			List[str]  							=      0
     while api_doc[pipeline_idx]["title"] != "Pipelines":
          pipeline_idx += 1

     __UpperCAmelCase     :			List[str]  							=      False
     __UpperCAmelCase     :			List[Any]  							=      api_doc[pipeline_idx]['''sections''']
     __UpperCAmelCase     :			Optional[Any]  							=      []

     # sort sub pipeline docs
     for pipeline_doc in pipeline_docs:
          if "section" in pipeline_doc:
               __UpperCAmelCase     :			int  							=      pipeline_doc['''section''']
               __UpperCAmelCase     :			str  							=      clean_doc_toc(_lowercase  )
               if overwrite:
                    __UpperCAmelCase     :			str  							=      new_sub_pipeline_doc
          new_pipeline_docs.append(_lowercase  )

     # sort overall pipeline doc
     __UpperCAmelCase     :			Optional[Any]  							=      clean_doc_toc(_lowercase  )

     if new_pipeline_docs != pipeline_docs:
          __UpperCAmelCase     :			Union[str, Any]  							=      True
          if overwrite:
               __UpperCAmelCase     :			Any  							=      new_pipeline_docs

     if diff:
          if overwrite:
               __UpperCAmelCase     :			Optional[int]  							=      api_doc
               with open(_lowercase      , '''w'''      , encoding='''utf-8'''  ) as f:
                    f.write(yaml.dump(_lowercase      , allow_unicode=_lowercase  )  )
          else:
               raise ValueError(
                   '''The model doc part of the table of content is not properly sorted, run `make style` to fix this.'''  )


if __name__ == "__main__":
      __UpperCAmelCase      :List[Any]			    =       argparse.ArgumentParser()
      parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
      __UpperCAmelCase      :List[str]			    =       parser.parse_args()

      check_scheduler_doc(args.fix_and_overwrite)
      check_pipeline_doc(args.fix_and_overwrite)



'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


__UpperCAmelCase      :Dict			    =       logging.get_logger(__name__)

__UpperCAmelCase      :str			    =       {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase      :Optional[int]			    =       {
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

__UpperCAmelCase      :Union[str, Any]			    =       {
    "bert-base-uncased": 5_1_2,
    "bert-large-uncased": 5_1_2,
    "bert-base-cased": 5_1_2,
    "bert-large-cased": 5_1_2,
    "bert-base-multilingual-uncased": 5_1_2,
    "bert-base-multilingual-cased": 5_1_2,
    "bert-base-chinese": 5_1_2,
    "bert-base-german-cased": 5_1_2,
    "bert-large-uncased-whole-word-masking": 5_1_2,
    "bert-large-cased-whole-word-masking": 5_1_2,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 5_1_2,
    "bert-large-cased-whole-word-masking-finetuned-squad": 5_1_2,
    "bert-base-cased-finetuned-mrpc": 5_1_2,
    "bert-base-german-dbmdz-cased": 5_1_2,
    "bert-base-german-dbmdz-uncased": 5_1_2,
    "TurkuNLP/bert-base-finnish-cased-v1": 5_1_2,
    "TurkuNLP/bert-base-finnish-uncased-v1": 5_1_2,
    "wietsedv/bert-base-dutch-cased": 5_1_2,
}

__UpperCAmelCase      :str			    =       {
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}



class      a    (		_a	):


     """simple docstring"""






     SCREAMING_SNAKE_CASE	:			Optional[int]							       =       VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE	:			List[Any]							       =       PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE	:			Tuple							       =       PRETRAINED_INIT_CONFIGURATION
     SCREAMING_SNAKE_CASE	:			Optional[int]							       =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE	:			Optional[Any]							       =       BertTokenizer

     def __init__(    self					:		Optional[int]							,			snake_case					:		Union[str, Any]=None							,			snake_case					:		str=None							,			snake_case					:		Any=True							,			snake_case					:		Tuple="[UNK]"							,			snake_case					:		int="[SEP]"							,			snake_case					:		Optional[Any]="[PAD]"							,			snake_case					:		int="[CLS]"							,			snake_case					:		Optional[Any]="[MASK]"							,			snake_case					:		Union[str, Any]=True							,			snake_case					:		List[Any]=None							,			**snake_case					:		int							,			)						->							str:
          super().__init__(
              snake_case							,			tokenizer_file=snake_case							,			do_lower_case=snake_case							,			unk_token=snake_case							,			sep_token=snake_case							,			pad_token=snake_case							,			cls_token=snake_case							,			mask_token=snake_case							,			tokenize_chinese_chars=snake_case							,			strip_accents=snake_case							,			**snake_case							,			)

          __UpperCAmelCase     :			List[str]  							=      json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
          if (
              normalizer_state.get('''lowercase'''							,			snake_case      ) != do_lower_case
              or normalizer_state.get('''strip_accents'''							,			snake_case      ) != strip_accents
              or normalizer_state.get('''handle_chinese_chars'''							,			snake_case      ) != tokenize_chinese_chars
          ):
               __UpperCAmelCase     :			List[Any]  							=      getattr(snake_case							,			normalizer_state.pop('''type'''      )      )
               __UpperCAmelCase     :			List[Any]  							=      do_lower_case
               __UpperCAmelCase     :			List[Any]  							=      strip_accents
               __UpperCAmelCase     :			str  							=      tokenize_chinese_chars
               __UpperCAmelCase     :			List[str]  							=      normalizer_class(**snake_case      )

          __UpperCAmelCase     :			Optional[int]  							=      do_lower_case

     def 			lowerCamelCase__			(    self					:		List[Any]							,			snake_case					:		Tuple							,			snake_case					:		List[str]=None      )						->							str:
          __UpperCAmelCase     :			Optional[int]  							=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output

     def 			lowerCamelCase__			(    self					:		str							,			snake_case					:		List[int]							,			snake_case					:		Optional[List[int]] = None      )						->							List[int]:
          __UpperCAmelCase     :			List[str]  							=      [self.sep_token_id]
          __UpperCAmelCase     :			Tuple  							=      [self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep      ) * [0]
          return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






     def 			lowerCamelCase__			(    self					:		List[Any]							,			snake_case					:		str							,			snake_case					:		Optional[str] = None      )						->							Tuple[str]:
          __UpperCAmelCase     :			Optional[int]  							=      self._tokenizer.model.save(snake_case							,			name=snake_case      )
          return tuple(snake_case      )