





'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


A__    :       Tuple              =logging.get_logger(__name__)

A__    :       Dict              ={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A__    :       List[Any]              ={
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

A__    :       Any              ={
    '''bert-base-uncased''': 5_12,
    '''bert-large-uncased''': 5_12,
    '''bert-base-cased''': 5_12,
    '''bert-large-cased''': 5_12,
    '''bert-base-multilingual-uncased''': 5_12,
    '''bert-base-multilingual-cased''': 5_12,
    '''bert-base-chinese''': 5_12,
    '''bert-base-german-cased''': 5_12,
    '''bert-large-uncased-whole-word-masking''': 5_12,
    '''bert-large-cased-whole-word-masking''': 5_12,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-base-cased-finetuned-mrpc''': 5_12,
    '''bert-base-german-dbmdz-cased''': 5_12,
    '''bert-base-german-dbmdz-uncased''': 5_12,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 5_12,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 5_12,
    '''wietsedv/bert-base-dutch-cased''': 5_12,
}

A__    :       Union[str, Any]              ={
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}




class 			UpperCAmelCase					(      snake_case_       ):
      _lowercase:      Dict											=       VOCAB_FILES_NAMES
      _lowercase:      Optional[Any]											=       PRETRAINED_VOCAB_FILES_MAP
      _lowercase:      int											=       PRETRAINED_INIT_CONFIGURATION
      _lowercase:      Any											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase:      Dict											=       BertTokenizer




      def __init__(      self      :			Union[str, Any]      ,      __snake_case      :			List[Any]=None      ,      __snake_case      :			List[str]=None      ,      __snake_case      :			List[str]=True      ,      __snake_case      :			Optional[int]="[UNK]"      ,      __snake_case      :			Tuple="[SEP]"      ,      __snake_case      :			List[str]="[PAD]"      ,      __snake_case      :			List[str]="[CLS]"      ,      __snake_case      :			Dict="[MASK]"      ,      __snake_case      :			str=True      ,      __snake_case      :			Tuple=None      ,      **__snake_case      :			List[Any]      ,      )    ->				Optional[int]:
            super().__init__(
                __snake_case      ,      tokenizer_file=__snake_case      ,      do_lower_case=__snake_case      ,      unk_token=__snake_case      ,      sep_token=__snake_case      ,      pad_token=__snake_case      ,      cls_token=__snake_case      ,      mask_token=__snake_case      ,      tokenize_chinese_chars=__snake_case      ,      strip_accents=__snake_case      ,      **__snake_case      ,      )

            _lowerCAmelCase												=							json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
            if (
                normalizer_state.get("""lowercase"""      ,      __snake_case  ) != do_lower_case
                or normalizer_state.get("""strip_accents"""      ,      __snake_case  ) != strip_accents
                or normalizer_state.get("""handle_chinese_chars"""      ,      __snake_case  ) != tokenize_chinese_chars
            ):
                  _lowerCAmelCase												=							getattr(__snake_case      ,      normalizer_state.pop("""type"""  )  )
                  _lowerCAmelCase												=							do_lower_case
                  _lowerCAmelCase												=							strip_accents
                  _lowerCAmelCase												=							tokenize_chinese_chars
                  _lowerCAmelCase												=							normalizer_class(**__snake_case  )

            _lowerCAmelCase												=							do_lower_case




      def     lowercase__   (      self      :			List[Any]      ,      __snake_case      :			Optional[int]      ,      __snake_case      :			Optional[int]=None  )    ->				Optional[int]:
            _lowerCAmelCase												=							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

            if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

            return output




      def     lowercase__   (      self      :			str      ,      __snake_case      :			List[int]      ,      __snake_case      :			Optional[List[int]] = None  )    ->				List[int]:
            _lowerCAmelCase												=							[self.sep_token_id]
            _lowerCAmelCase												=							[self.cls_token_id]
            if token_ids_a is None:
                  return len(cls + token_ids_a + sep  ) * [0]
            return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




      def     lowercase__   (      self      :			List[Any]      ,      __snake_case      :			str      ,      __snake_case      :			Optional[str] = None  )    ->				Tuple[str]:
            _lowerCAmelCase												=							self._tokenizer.model.save(__snake_case      ,      name=__snake_case  )
            return tuple(__snake_case  )













'''simple docstring'''






import os
import tempfile
import unittest

from transformers.models.marian.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
from transformers.testing_utils import slow
from transformers.utils import cached_property




@unittest.skipUnless(os.path.exists(snake_case_       )							,		'''Tatoeba directory does not exist.'''       )
class 			UpperCAmelCase					(      unittest.TestCase       ):




      @cached_property
      def     lowercase__   (      self      :			int  )    ->				Any:
            _lowerCAmelCase												=							tempfile.mkdtemp()
            return TatoebaConverter(save_dir=__snake_case  )




      @slow
      def     lowercase__   (      self      :			Dict  )    ->				int:
            self.resolver.convert_models(["""heb-eng"""]  )




      @slow
      def     lowercase__   (      self      :			Optional[int]  )    ->				Tuple:
            _lowerCAmelCase					,							_lowerCAmelCase												=							self.resolver.write_model_card("""opus-mt-he-en"""      ,      dry_run=__snake_case  )
            assert mmeta["long_pair"] == "heb-eng"






