




'''simple docstring'''


import tempfile
import unittest

import numpy as np
from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax


if is_flax_available():
				import os

				from flax.core.frozen_dict import unfreeze
				from flax.traverse_util import flatten_dict

				from transformers import FlaxBertModel

				_snake_case				:				Optional[Any]       							=		'0.12'  # assumed parallelism: 8

@require_flax
@is_staging_test
class       A   (		unittest.TestCase   ):







							@classmethod
							def        __lowerCAmelCase      (  cls	:  Tuple )      ->							int:


														"""simple docstring"""


														_a     					=    TOKEN
														HfFolder.save_token(lowerCAmelCase_ )







							@classmethod
							def        __lowerCAmelCase      (  cls	:  Tuple )      ->							Optional[int]:


														"""simple docstring"""


														try:
																					delete_repo(token=cls._token							,      repo_id='''test-model-flax''' )
														except HTTPError:
																					pass

														try:
																					delete_repo(token=cls._token							,      repo_id='''valid_org/test-model-flax-org''' )
														except HTTPError:
																					pass







							def        __lowerCAmelCase      (  self	:  Any )      ->							Optional[Any]:


														"""simple docstring"""


														_a     					=    BertConfig(
														    vocab_size=99							,      hidden_size=32							,      num_hidden_layers=5							,      num_attention_heads=4							,      intermediate_size=37 )
														_a     					=    FlaxBertModel(lowerCAmelCase_ )
														model.push_to_hub('''test-model-flax'''							,      use_auth_token=self._token )

														_a     					=    FlaxBertModel.from_pretrained(F'{USER}/test-model-flax' )

														_a     					=    flatten_dict(unfreeze(model.params ) )
														_a     					=    flatten_dict(unfreeze(new_model.params ) )

														for key in base_params.keys():
																					_a     					=    (base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCAmelCase_							,      1e-3							,      msg=F'{key} not identical' )

														# Reset repo
														delete_repo(token=self._token							,      repo_id='''test-model-flax''' )

														# Push to hub via save_pretrained
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(lowerCAmelCase_							,      repo_id='''test-model-flax'''							,      push_to_hub=lowerCAmelCase_							,      use_auth_token=self._token )

														_a     					=    FlaxBertModel.from_pretrained(F'{USER}/test-model-flax' )

														_a     					=    flatten_dict(unfreeze(model.params ) )
														_a     					=    flatten_dict(unfreeze(new_model.params ) )

														for key in base_params.keys():
																					_a     					=    (base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCAmelCase_							,      1e-3							,      msg=F'{key} not identical' )







							def        __lowerCAmelCase      (  self	:  int )      ->							Tuple:


														"""simple docstring"""


														_a     					=    BertConfig(
														    vocab_size=99							,      hidden_size=32							,      num_hidden_layers=5							,      num_attention_heads=4							,      intermediate_size=37 )
														_a     					=    FlaxBertModel(lowerCAmelCase_ )
														model.push_to_hub('''valid_org/test-model-flax-org'''							,      use_auth_token=self._token )

														_a     					=    FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org''' )

														_a     					=    flatten_dict(unfreeze(model.params ) )
														_a     					=    flatten_dict(unfreeze(new_model.params ) )

														for key in base_params.keys():
																					_a     					=    (base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCAmelCase_							,      1e-3							,      msg=F'{key} not identical' )

														# Reset repo
														delete_repo(token=self._token							,      repo_id='''valid_org/test-model-flax-org''' )

														# Push to hub via save_pretrained
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(
																					    lowerCAmelCase_							,      repo_id='''valid_org/test-model-flax-org'''							,      push_to_hub=lowerCAmelCase_							,      use_auth_token=self._token )

														_a     					=    FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org''' )

														_a     					=    flatten_dict(unfreeze(model.params ) )
														_a     					=    flatten_dict(unfreeze(new_model.params ) )

														for key in base_params.keys():
																					_a     					=    (base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCAmelCase_							,      1e-3							,      msg=F'{key} not identical' )



def 		snake_case_   (UpperCamelCase	:  List[str]			,						UpperCamelCase	:  Optional[Any]				):





							'''simple docstring'''


							_a     					=    True
							_a     					=    flatten_dict(modela.params				)
							_a     					=    flatten_dict(modela.params				)
							for key in flat_params_a.keys():
														if np.sum(np.abs(flat_params_a[key] - flat_params_a[key]				)				) > 1e-4:
																					_a     					=    False

							return models_are_equal

@require_flax
class       A   (		unittest.TestCase   ):







							def        __lowerCAmelCase      (  self	:  Dict )      ->							int:


														"""simple docstring"""


														_a     					=    BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only''' )
														_a     					=    FlaxBertModel(lowerCAmelCase_ )

														_a     					=    '''bert'''
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(os.path.join(lowerCAmelCase_							,      lowerCAmelCase_ ) )

																					with self.assertRaises(lowerCAmelCase_ ):
																												_a     					=    FlaxBertModel.from_pretrained(lowerCAmelCase_ )

																					_a     					=    FlaxBertModel.from_pretrained(lowerCAmelCase_							,      subfolder=lowerCAmelCase_ )

														self.assertTrue(check_models_equal(lowerCAmelCase_							,      lowerCAmelCase_ ) )







							def        __lowerCAmelCase      (  self	:  Union[str, Any] )      ->							Optional[int]:


														"""simple docstring"""


														_a     					=    BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only''' )
														_a     					=    FlaxBertModel(lowerCAmelCase_ )

														_a     					=    '''bert'''
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(os.path.join(lowerCAmelCase_							,      lowerCAmelCase_ )							,      max_shard_size='''10KB''' )

																					with self.assertRaises(lowerCAmelCase_ ):
																												_a     					=    FlaxBertModel.from_pretrained(lowerCAmelCase_ )

																					_a     					=    FlaxBertModel.from_pretrained(lowerCAmelCase_							,      subfolder=lowerCAmelCase_ )

														self.assertTrue(check_models_equal(lowerCAmelCase_							,      lowerCAmelCase_ ) )







							def        __lowerCAmelCase      (  self	:  Optional[Any] )      ->							List[str]:


														"""simple docstring"""


														_a     					=    '''bert'''
														_a     					=    '''hf-internal-testing/tiny-random-bert-subfolder'''

														with self.assertRaises(lowerCAmelCase_ ):
																					_a     					=    FlaxBertModel.from_pretrained(lowerCAmelCase_ )

														_a     					=    FlaxBertModel.from_pretrained(lowerCAmelCase_							,      subfolder=lowerCAmelCase_ )

														self.assertIsNotNone(lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Tuple )      ->							Tuple:


														"""simple docstring"""


														_a     					=    '''bert'''
														_a     					=    '''hf-internal-testing/tiny-random-bert-sharded-subfolder'''
														with self.assertRaises(lowerCAmelCase_ ):
																					_a     					=    FlaxBertModel.from_pretrained(lowerCAmelCase_ )

														_a     					=    FlaxBertModel.from_pretrained(lowerCAmelCase_							,      subfolder=lowerCAmelCase_ )

														self.assertIsNotNone(lowerCAmelCase_ )







'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_snake_case				:				Any       							=		logging.get_logger(__name__)

_snake_case				:				Any       							=		{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case				:				List[str]       							=		{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

_snake_case				:				int       							=		{
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

_snake_case				:				int       							=		{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}

class       A   (		_a   ):
							lowercase_          =    VOCAB_FILES_NAMES
							lowercase_          =    PRETRAINED_VOCAB_FILES_MAP
							lowercase_          =    PRETRAINED_INIT_CONFIGURATION
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_          =    BertTokenizer







							def __init__(  self	:  str							,      lowerCAmelCase_	:  Union[str, Any]=None							,      lowerCAmelCase_	:  str=None							,      lowerCAmelCase_	:  List[Any]=True							,      lowerCAmelCase_	:  Any="[UNK]"							,      lowerCAmelCase_	:  Union[str, Any]="[SEP]"							,      lowerCAmelCase_	:  Tuple="[PAD]"							,      lowerCAmelCase_	:  Tuple="[CLS]"							,      lowerCAmelCase_	:  Optional[int]="[MASK]"							,      lowerCAmelCase_	:  Dict=True							,      lowerCAmelCase_	:  Union[str, Any]=None							,      **lowerCAmelCase_	:  Union[str, Any]							,      )      ->							Union[str, Any]:


														"""simple docstring"""


														super().__init__(
														    lowerCAmelCase_							,      tokenizer_file=lowerCAmelCase_							,      do_lower_case=lowerCAmelCase_							,      unk_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      tokenize_chinese_chars=lowerCAmelCase_							,      strip_accents=lowerCAmelCase_							,      **lowerCAmelCase_							,      )

														_a     					=    json.loads(self.backend_tokenizer.normalizer.__getstate__() )
														if (
														    normalizer_state.get('''lowercase'''							,      lowerCAmelCase_ ) != do_lower_case
														    or normalizer_state.get('''strip_accents'''							,      lowerCAmelCase_ ) != strip_accents
														    or normalizer_state.get('''handle_chinese_chars'''							,      lowerCAmelCase_ ) != tokenize_chinese_chars
														):
																					_a     					=    getattr(lowerCAmelCase_							,      normalizer_state.pop('''type''' ) )
																					_a     					=    do_lower_case
																					_a     					=    strip_accents
																					_a     					=    tokenize_chinese_chars
																					_a     					=    normalizer_class(**lowerCAmelCase_ )

														_a     					=    do_lower_case







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  Dict							,      lowerCAmelCase_	:  int=None )      ->							List[str]:


														"""simple docstring"""


														_a     					=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep ) * [0]
														return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[str] = None )      ->							Tuple[str]:


														"""simple docstring"""


														_a     					=    self._tokenizer.model.save(lowerCAmelCase_							,      name=lowerCAmelCase_ )
														return tuple(lowerCAmelCase_ )

