





"""simple docstring"""




import unittest

from transformers import BertGenerationTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case           =			"""▁"""

_snake_case           =			get_tests_dir("""fixtures/test_sentencepiece.model""")





@require_sentencepiece
class  lowerCAmelCase							(       lowercase_							,							unittest.TestCase ):
  __lowerCamelCase		=      BertGenerationTokenizer
  __lowerCamelCase		=      False
  __lowerCamelCase		=      True







  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     super().setUp()

     lowercase__			       =  BertGenerationTokenizer(_lowercase   ,							keep_accents=_lowercase )
     tokenizer.save_pretrained(self.tmpdirname )







  def   UpperCAmelCase					(      self  :Dict ):






     '''simple docstring'''





     lowercase__			       =  "<s>"
     lowercase__			       =  1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowercase )   ,							_lowercase )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowercase )   ,							_lowercase )







  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     lowercase__			       =  list(self.get_tokenizer().get_vocab().keys() )

     self.assertEqual(vocab_keys[0]   ,							"<unk>" )
     self.assertEqual(vocab_keys[1]   ,							"<s>" )
     self.assertEqual(vocab_keys[-1]   ,							"<pad>" )
     self.assertEqual(len(_lowercase )   ,							10_02 )







  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     self.assertEqual(self.get_tokenizer().vocab_size   ,							10_00 )







  def   UpperCAmelCase					(      self  :Union[str, Any] ):






     '''simple docstring'''





     lowercase__			       =  BertGenerationTokenizer(_lowercase   ,							keep_accents=_lowercase )

     lowercase__			       =  tokenizer.tokenize("This is a test" )
     self.assertListEqual(_lowercase   ,							["▁This", "▁is", "▁a", "▁t", "est"] )

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(_lowercase )   ,							[2_85, 46, 10, 1_70, 3_82]   ,							)

     lowercase__			       =  tokenizer.tokenize("I was born in 92000, and this is falsé." )
     self.assertListEqual(
         _lowercase   ,							[
             SPIECE_UNDERLINE + "I",
             SPIECE_UNDERLINE + "was",
             SPIECE_UNDERLINE + "b",
             "or",
             "n",
             SPIECE_UNDERLINE + "in",
             SPIECE_UNDERLINE + "",
             "9",
             "2",
             "0",
             "0",
             "0",
             ",",
             SPIECE_UNDERLINE + "and",
             SPIECE_UNDERLINE + "this",
             SPIECE_UNDERLINE + "is",
             SPIECE_UNDERLINE + "f",
             "al",
             "s",
             "é",
             ".",
         ]   ,							)
     lowercase__			       =  tokenizer.convert_tokens_to_ids(_lowercase )
     self.assertListEqual(
         _lowercase   ,							[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]   ,							)

     lowercase__			       =  tokenizer.convert_ids_to_tokens(_lowercase )
     self.assertListEqual(
         _lowercase   ,							[
             SPIECE_UNDERLINE + "I",
             SPIECE_UNDERLINE + "was",
             SPIECE_UNDERLINE + "b",
             "or",
             "n",
             SPIECE_UNDERLINE + "in",
             SPIECE_UNDERLINE + "",
             "<unk>",
             "2",
             "0",
             "0",
             "0",
             ",",
             SPIECE_UNDERLINE + "and",
             SPIECE_UNDERLINE + "this",
             SPIECE_UNDERLINE + "is",
             SPIECE_UNDERLINE + "f",
             "al",
             "s",
             "<unk>",
             ".",
         ]   ,							)







  @cached_property
  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     return BertGenerationTokenizer.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder" )







  @slow
  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     lowercase__			       =  "Hello World!"
     lowercase__			       =  [1_85_36, 22_60, 1_01]

     self.assertListEqual(_lowercase   ,							self.big_tokenizer.encode(_lowercase ) )







  @slow
  def   UpperCAmelCase					(      self  :Dict ):






     '''simple docstring'''





     lowercase__			       =  (
         "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
         " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
     )
     lowercase__			       =  [
         8_71,
         4_19,
         3_58,
         9_46,
         9_91,
         25_21,
         4_52,
         3_58,
         13_57,
         3_87,
         77_51,
         35_36,
         1_12,
         9_85,
         4_56,
         1_26,
         8_65,
         9_38,
         54_00,
         57_34,
         4_58,
         13_68,
         4_67,
         7_86,
         24_62,
         52_46,
         11_59,
         6_33,
         8_65,
         45_19,
         4_57,
         5_82,
         8_52,
         25_57,
         4_27,
         9_16,
         5_08,
         4_05,
         3_43_24,
         4_97,
         3_91,
         4_08,
         1_13_42,
         12_44,
         3_85,
         1_00,
         9_38,
         9_85,
         4_56,
         5_74,
         3_62,
         1_25_97,
         32_00,
         31_29,
         11_72,
     ]

     self.assertListEqual(_lowercase   ,							self.big_tokenizer.encode(_lowercase ) )







  @require_torch
  @slow
  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     import torch

     from transformers import BertGenerationConfig, BertGenerationEncoder

     # Build sequence
     lowercase__			       =  list(self.big_tokenizer.get_vocab().keys() )[:10]
     lowercase__			       =  " ".join(_lowercase )
     lowercase__			       =  self.big_tokenizer.encode_plus(_lowercase   ,							return_tensors="pt"   ,							return_token_type_ids=_lowercase )
     lowercase__			       =  self.big_tokenizer.batch_encode_plus(
         [sequence + " " + sequence]   ,							return_tensors="pt"   ,							return_token_type_ids=_lowercase )

     lowercase__			       =  BertGenerationConfig()
     lowercase__			       =  BertGenerationEncoder(_lowercase )

     assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

     with torch.no_grad():
        model(**_lowercase )
        model(**_lowercase )







  @slow
  def   UpperCAmelCase					(      self  :List[str] ):






     '''simple docstring'''





     lowercase__			       =  {"input_ids": [[3_92_86, 4_58, 3_63_35, 20_01, 4_56, 1_30_73, 1_32_66, 4_55, 1_13, 77_46, 17_41, 1_11_57, 3_91, 1_30_73, 1_32_66, 4_55, 1_13, 39_67, 3_54_12, 1_13, 49_36, 1_09, 38_70, 23_77, 1_13, 3_00_84, 4_57_20, 4_58, 1_34, 1_74_96, 1_12, 5_03, 1_16_72, 1_13, 1_18, 1_12, 56_65, 1_33_47, 3_86_87, 1_12, 14_96, 3_13_89, 1_12, 32_68, 4_72_64, 1_34, 9_62, 1_12, 1_63_77, 80_35, 2_31_30, 4_30, 1_21_69, 1_55_18, 2_85_92, 4_58, 1_46, 4_16_97, 1_09, 3_91, 1_21_69, 1_55_18, 1_66_89, 4_58, 1_46, 4_13_58, 1_09, 4_52, 7_26, 40_34, 1_11, 7_63, 3_54_12, 50_82, 3_88, 19_03, 1_11, 90_51, 3_91, 28_70, 4_89_18, 19_00, 11_23, 5_50, 9_98, 1_12, 95_86, 1_59_85, 4_55, 3_91, 4_10, 2_29_55, 3_76_36, 1_14], [4_48, 1_74_96, 4_19, 36_63, 3_85, 7_63, 1_13, 2_75_33, 28_70, 32_83, 1_30_43, 16_39, 2_47_13, 5_23, 6_56, 2_40_13, 1_85_50, 25_21, 5_17, 2_70_14, 2_12_44, 4_20, 12_12, 14_65, 3_91, 9_27, 48_33, 3_88, 5_78, 1_17_86, 1_14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4_84, 21_69, 76_87, 2_19_32, 1_81_46, 7_26, 3_63, 1_70_32, 33_91, 1_14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=_lowercase   ,							model_name="google/bert_for_seq_generation_L-24_bbc_encoder"   ,							revision="c817d1fd1be2ffa69431227a1fe320544943d4db"   ,							)













import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
  from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm


def _A       (   __magic_name__	):
   if is_torch_version("<"  , "2.0.0"	) or not hasattr(__magic_name__  , "_dynamo"	):
      return False
   return isinstance(__magic_name__  , torch._dynamo.eval_frame.OptimizedModule	)


def _A       (   __magic_name__  , __magic_name__ = True	):
   lowercase__			       =  (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

   lowercase__			       =  is_compiled_module(__magic_name__	)
   if is_compiled:
      lowercase__			       =  model
      lowercase__			       =  model._orig_mod

   if is_deepspeed_available():
      options += (DeepSpeedEngine,)

   while isinstance(__magic_name__  , __magic_name__	):
      lowercase__			       =  model.module

   if not keep_fpaa_wrapper:
      lowercase__			       =  getattr(__magic_name__  , "forward"	)
      lowercase__			       =  model.__dict__.pop("_original_forward"  , __magic_name__	)
      if original_forward is not None:
         while hasattr(__magic_name__  , "__wrapped__"	):
            lowercase__			       =  forward.__wrapped__
            if forward == original_forward:
               break
         lowercase__			       =  forward
      if getattr(__magic_name__  , "_converted_to_transformer_engine"  , __magic_name__	):
         convert_model(__magic_name__  , to_transformer_engine=__magic_name__	)

   if is_compiled:
      lowercase__			       =  model
      lowercase__			       =  compiled_model

   return model


def _A       (   ):
   PartialState().wait_for_everyone()


def _A       (   __magic_name__  , __magic_name__	):
   if PartialState().distributed_type == DistributedType.TPU:
      xm.save(__magic_name__  , __magic_name__	)
   elif PartialState().local_process_index == 0:
      torch.save(__magic_name__  , __magic_name__	)


@contextmanager
def _A       (   **__magic_name__	):
   for key, value in kwargs.items():
      lowercase__			       =  str(__magic_name__	)

   yield

   for key in kwargs:
      if key.upper() in os.environ:
         del os.environ[key.upper()]


def _A       (   __magic_name__	):
   if not hasattr(__magic_name__  , "__qualname__"	) and not hasattr(__magic_name__  , "__name__"	):
      lowercase__			       =  getattr(__magic_name__  , "__class__"  , __magic_name__	)
   if hasattr(__magic_name__  , "__qualname__"	):
      return obj.__qualname__
   if hasattr(__magic_name__  , "__name__"	):
      return obj.__name__
   return str(__magic_name__	)


def _A       (   __magic_name__  , __magic_name__	):
   for key, value in source.items():
      if isinstance(__magic_name__  , __magic_name__	):
         lowercase__			       =  destination.setdefault(__magic_name__  , {}	)
         merge_dicts(__magic_name__  , __magic_name__	)
      else:
         lowercase__			       =  value

   return destination







def _A       (   __magic_name__ = None	):
   if port is None:
      lowercase__			       =  2_9500
   with socket.socket(socket.AF_INET  , socket.SOCK_STREAM	) as s:
      return s.connect_ex(("localhost", port)	) == 0






