'''simple docstring'''






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class 			UpperCAmelCase					(      lowerCAmelCase__							,		unittest.TestCase       ):
      _lowercase:      Optional[Any]											=       BertJapaneseTokenizer
      _lowercase:      int											=       False
      _lowercase:      Optional[int]											=       True




      def     lowercase__   (      self      :			Union[str, Any]  )    ->				str:
            super().setUp()

            _lowerCAmelCase												=							[
                """[UNK]""",
                """[CLS]""",
                """[SEP]""",
                """こんにちは""",
                """こん""",
                """にちは""",
                """ばんは""",
                """##こん""",
                """##にちは""",
                """##ばんは""",
                """世界""",
                """##世界""",
                """、""",
                """##、""",
                """。""",
                """##。""",
            ]

            _lowerCAmelCase												=							os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES["""vocab_file"""]  )
            with open(self.vocab_file      ,      """w"""      ,      encoding="""utf-8"""  ) as vocab_writer:
                  vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]  )  )




      def     lowercase__   (      self      :			Dict      ,      __snake_case      :			Dict  )    ->				Any:
            _lowerCAmelCase												=							"""こんにちは、世界。 \nこんばんは、世界。"""
            _lowerCAmelCase												=							"""こんにちは 、 世界 。 こんばんは 、 世界 。"""
            return input_text, output_text




      def     lowercase__   (      self      :			Any      ,      __snake_case      :			Dict  )    ->				Dict:
            _lowerCAmelCase					,							_lowerCAmelCase												=							self.get_input_output_texts(a__  )
            _lowerCAmelCase												=							tokenizer.encode(a__      ,      add_special_tokens=a__  )
            _lowerCAmelCase												=							tokenizer.decode(a__      ,      clean_up_tokenization_spaces=a__  )
            return text, ids




      def     lowercase__   (      self      :			str  )    ->				Optional[Any]:
            pass  # TODO add if relevant




      def     lowercase__   (      self      :			Tuple  )    ->				Optional[int]:
            pass  # TODO add if relevant




      def     lowercase__   (      self      :			Dict  )    ->				Union[str, Any]:
            pass  # TODO add if relevant




      def     lowercase__   (      self      :			Dict  )    ->				Dict:
            _lowerCAmelCase												=							self.tokenizer_class(self.vocab_file  )

            _lowerCAmelCase												=							tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""  )
            self.assertListEqual(a__      ,      ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]  )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(a__  )      ,      [3, 12, 10, 14, 4, 9, 12, 10, 14]  )




      def     lowercase__   (      self      :			Tuple  )    ->				Optional[int]:
            _lowerCAmelCase												=							self.tokenizer_class(self.vocab_file      ,      word_tokenizer_type="""mecab"""  )
            self.assertIsNotNone(a__  )

            _lowerCAmelCase												=							"""こんにちは、世界。\nこんばんは、世界。"""
            _lowerCAmelCase												=							tokenizer.tokenize(a__  )
            self.assertListEqual(a__      ,      ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]  )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(a__  )      ,      [3, 12, 10, 14, 4, 9, 12, 10, 14]  )

            _lowerCAmelCase												=							os.path.join(self.tmpdirname      ,      """tokenizer.bin"""  )
            with open(a__      ,      """wb"""  ) as handle:
                  pickle.dump(a__      ,      a__  )

            with open(a__      ,      """rb"""  ) as handle:
                  _lowerCAmelCase												=							pickle.load(a__  )

            _lowerCAmelCase												=							tokenizer_new.tokenize(a__  )

            self.assertListEqual(a__      ,      a__  )




      def     lowercase__   (      self      :			Any  )    ->				Tuple:
            _lowerCAmelCase												=							MecabTokenizer(mecab_dic="""ipadic"""  )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,      )




      def     lowercase__   (      self      :			List[Any]  )    ->				Optional[Any]:
            try:
                  _lowerCAmelCase												=							MecabTokenizer(mecab_dic="""unidic_lite"""  )
            except ModuleNotFoundError:
                  return

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,      )




      def     lowercase__   (      self      :			Optional[int]  )    ->				Optional[int]:
            try:
                  _lowerCAmelCase												=							MecabTokenizer(mecab_dic="""unidic"""  )
            except ModuleNotFoundError:
                  return

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,      )




      def     lowercase__   (      self      :			int  )    ->				Optional[int]:
            _lowerCAmelCase												=							MecabTokenizer(do_lower_case=a__      ,      mecab_dic="""ipadic"""  )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,      )




      def     lowercase__   (      self      :			Optional[int]  )    ->				Optional[int]:
            try:
                  _lowerCAmelCase												=							MecabTokenizer(
                      do_lower_case=a__      ,      normalize_text=a__      ,      mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""  )
            except RuntimeError:
                  # if dict doesn't exist in the system, previous code raises this error.
                  return

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,      )




      def     lowercase__   (      self      :			Tuple  )    ->				Tuple:
            _lowerCAmelCase												=							MecabTokenizer(normalize_text=a__      ,      mecab_dic="""ipadic"""  )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]      ,      )




      @require_sudachi
      def     lowercase__   (      self      :			Any  )    ->				Optional[int]:
            _lowerCAmelCase												=							self.tokenizer_class(self.vocab_file      ,      word_tokenizer_type="""sudachi"""  )
            self.assertIsNotNone(a__  )

            _lowerCAmelCase												=							"""こんにちは、世界。\nこんばんは、世界。"""
            _lowerCAmelCase												=							tokenizer.tokenize(a__  )
            self.assertListEqual(a__      ,      ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]  )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(a__  )      ,      [3, 12, 10, 14, 4, 9, 12, 10, 14]  )

            _lowerCAmelCase												=							os.path.join(self.tmpdirname      ,      """tokenizer.bin"""  )
            with open(a__      ,      """wb"""  ) as handle:
                  pickle.dump(a__      ,      a__  )

            with open(a__      ,      """rb"""  ) as handle:
                  _lowerCAmelCase												=							pickle.load(a__  )

            _lowerCAmelCase												=							tokenizer_new.tokenize(a__  )

            self.assertListEqual(a__      ,      a__  )




      @require_sudachi
      def     lowercase__   (      self      :			Optional[int]  )    ->				Union[str, Any]:
            _lowerCAmelCase												=							SudachiTokenizer(sudachi_dict_type="""core"""  )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      [""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]      ,      )




      @require_sudachi
      def     lowercase__   (      self      :			Optional[Any]  )    ->				List[str]:
            _lowerCAmelCase												=							SudachiTokenizer(sudachi_dict_type="""core"""      ,      sudachi_split_mode="""A"""  )

            self.assertListEqual(tokenizer.tokenize("""外国人参政権"""  )      ,      ["""外国""", """人""", """参政""", """権"""]  )




      @require_sudachi
      def     lowercase__   (      self      :			int  )    ->				int:
            _lowerCAmelCase												=							SudachiTokenizer(sudachi_dict_type="""core"""      ,      sudachi_split_mode="""B"""  )

            self.assertListEqual(tokenizer.tokenize("""外国人参政権"""  )      ,      ["""外国人""", """参政権"""]  )




      @require_sudachi
      def     lowercase__   (      self      :			List[str]  )    ->				Optional[Any]:
            _lowerCAmelCase												=							SudachiTokenizer(sudachi_dict_type="""core"""      ,      sudachi_split_mode="""C"""  )

            self.assertListEqual(tokenizer.tokenize("""外国人参政権"""  )      ,      ["""外国人参政権"""]  )




      @require_sudachi
      def     lowercase__   (      self      :			Optional[int]  )    ->				Tuple:
            _lowerCAmelCase												=							SudachiTokenizer(do_lower_case=a__      ,      sudachi_dict_type="""core"""  )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      [""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]      ,      )




      @require_sudachi
      def     lowercase__   (      self      :			Tuple  )    ->				Tuple:
            _lowerCAmelCase												=							SudachiTokenizer(normalize_text=a__      ,      sudachi_dict_type="""core"""  )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      [""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]      ,      )




      @require_sudachi
      def     lowercase__   (      self      :			Any  )    ->				List[Any]:
            _lowerCAmelCase												=							SudachiTokenizer(trim_whitespace=a__      ,      sudachi_dict_type="""core"""  )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,      )




      @require_jumanpp
      def     lowercase__   (      self      :			Any  )    ->				str:
            _lowerCAmelCase												=							self.tokenizer_class(self.vocab_file      ,      word_tokenizer_type="""jumanpp"""  )
            self.assertIsNotNone(a__  )

            _lowerCAmelCase												=							"""こんにちは、世界。\nこんばんは、世界。"""
            _lowerCAmelCase												=							tokenizer.tokenize(a__  )
            self.assertListEqual(a__      ,      ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]  )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(a__  )      ,      [3, 12, 10, 14, 4, 9, 12, 10, 14]  )

            _lowerCAmelCase												=							os.path.join(self.tmpdirname      ,      """tokenizer.bin"""  )
            with open(a__      ,      """wb"""  ) as handle:
                  pickle.dump(a__      ,      a__  )

            with open(a__      ,      """rb"""  ) as handle:
                  _lowerCAmelCase												=							pickle.load(a__  )

            _lowerCAmelCase												=							tokenizer_new.tokenize(a__  )

            self.assertListEqual(a__      ,      a__  )




      @require_jumanpp
      def     lowercase__   (      self      :			Any  )    ->				int:
            _lowerCAmelCase												=							JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,      )




      @require_jumanpp
      def     lowercase__   (      self      :			Any  )    ->				str:
            _lowerCAmelCase												=							JumanppTokenizer(do_lower_case=a__  )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,      )




      @require_jumanpp
      def     lowercase__   (      self      :			Tuple  )    ->				Tuple:
            _lowerCAmelCase												=							JumanppTokenizer(normalize_text=a__  )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,      )




      @require_jumanpp
      def     lowercase__   (      self      :			Tuple  )    ->				Optional[int]:
            _lowerCAmelCase												=							JumanppTokenizer(trim_whitespace=a__  )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """  )      ,      ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]      ,      )




      @require_jumanpp
      def     lowercase__   (      self      :			Dict  )    ->				str:
            _lowerCAmelCase												=							JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""  )      ,      ["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]      ,      )




      def     lowercase__   (      self      :			Dict  )    ->				int:
            _lowerCAmelCase												=							["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

            _lowerCAmelCase												=							{}
            for i, token in enumerate(a__  ):
                  _lowerCAmelCase												=							i
            _lowerCAmelCase												=							WordpieceTokenizer(vocab=a__      ,      unk_token="""[UNK]"""  )

            self.assertListEqual(tokenizer.tokenize(""""""  )      ,      []  )

            self.assertListEqual(tokenizer.tokenize("""こんにちは"""  )      ,      ["""こんにちは"""]  )

            self.assertListEqual(tokenizer.tokenize("""こんばんは"""  )      ,      ["""こん""", """##ばんは"""]  )

            self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""  )      ,      ["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]  )




      def     lowercase__   (      self      :			Union[str, Any]  )    ->				str:
            _lowerCAmelCase												=							BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""  )
            _lowerCAmelCase												=							tokenizer.subword_tokenizer

            _lowerCAmelCase												=							subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""  )
            self.assertListEqual(a__      ,      ["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]  )

            _lowerCAmelCase												=							subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""  )
            self.assertListEqual(a__      ,      ["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]  )




      def     lowercase__   (      self      :			str  )    ->				Union[str, Any]:
            _lowerCAmelCase												=							self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""  )

            _lowerCAmelCase												=							tokenizer.encode("""ありがとう。"""      ,      add_special_tokens=a__  )
            _lowerCAmelCase												=							tokenizer.encode("""どういたしまして。"""      ,      add_special_tokens=a__  )

            _lowerCAmelCase												=							tokenizer.build_inputs_with_special_tokens(a__  )
            _lowerCAmelCase												=							tokenizer.build_inputs_with_special_tokens(a__      ,      a__  )

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			UpperCAmelCase					(      lowerCAmelCase__							,		unittest.TestCase       ):
      _lowercase:      Tuple											=       BertJapaneseTokenizer
      _lowercase:      str											=       False




      def     lowercase__   (      self      :			Union[str, Any]  )    ->				str:
            super().setUp()

            _lowerCAmelCase												=							["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

            _lowerCAmelCase												=							os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES["""vocab_file"""]  )
            with open(self.vocab_file      ,      """w"""      ,      encoding="""utf-8"""  ) as vocab_writer:
                  vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]  )  )




      def     lowercase__   (      self      :			Optional[Any]      ,      **__snake_case      :			List[Any]  )    ->				Any:
            return BertJapaneseTokenizer.from_pretrained(self.tmpdirname      ,      subword_tokenizer_type="""character"""      ,      **a__  )




      def     lowercase__   (      self      :			Optional[Any]      ,      __snake_case      :			Dict  )    ->				List[str]:
            _lowerCAmelCase												=							"""こんにちは、世界。 \nこんばんは、世界。"""
            _lowerCAmelCase												=							"""こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
            return input_text, output_text




      def     lowercase__   (      self      :			List[str]  )    ->				str:
            pass  # TODO add if relevant




      def     lowercase__   (      self      :			Optional[Any]  )    ->				List[Any]:
            pass  # TODO add if relevant




      def     lowercase__   (      self      :			Any  )    ->				List[str]:
            pass  # TODO add if relevant




      def     lowercase__   (      self      :			int  )    ->				Optional[int]:
            _lowerCAmelCase												=							self.tokenizer_class(self.vocab_file      ,      subword_tokenizer_type="""character"""  )

            _lowerCAmelCase												=							tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""  )
            self.assertListEqual(
                a__      ,      ["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]  )
            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(a__  )      ,      [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]  )




      def     lowercase__   (      self      :			Optional[int]  )    ->				List[Any]:
            _lowerCAmelCase												=							["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

            _lowerCAmelCase												=							{}
            for i, token in enumerate(a__  ):
                  _lowerCAmelCase												=							i
            _lowerCAmelCase												=							CharacterTokenizer(vocab=a__      ,      unk_token="""[UNK]"""  )

            self.assertListEqual(tokenizer.tokenize(""""""  )      ,      []  )

            self.assertListEqual(tokenizer.tokenize("""こんにちは"""  )      ,      ["""こ""", """ん""", """に""", """ち""", """は"""]  )

            self.assertListEqual(tokenizer.tokenize("""こんにちほ"""  )      ,      ["""こ""", """ん""", """に""", """ち""", """[UNK]"""]  )




      def     lowercase__   (      self      :			List[Any]  )    ->				Tuple:
            _lowerCAmelCase												=							self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""  )

            _lowerCAmelCase												=							tokenizer.encode("""ありがとう。"""      ,      add_special_tokens=a__  )
            _lowerCAmelCase												=							tokenizer.encode("""どういたしまして。"""      ,      add_special_tokens=a__  )

            _lowerCAmelCase												=							tokenizer.build_inputs_with_special_tokens(a__  )
            _lowerCAmelCase												=							tokenizer.build_inputs_with_special_tokens(a__      ,      a__  )

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			UpperCAmelCase					(      unittest.TestCase       ):




      def     lowercase__   (      self      :			Dict  )    ->				Union[str, Any]:
            _lowerCAmelCase												=							"""cl-tohoku/bert-base-japanese"""
            _lowerCAmelCase												=							AutoTokenizer.from_pretrained(a__  )
            self.assertIsInstance(a__      ,      a__  )




class 			UpperCAmelCase					(      unittest.TestCase       ):




      def     lowercase__   (      self      :			Optional[int]  )    ->				Tuple:
            _lowerCAmelCase												=							"""cl-tohoku/bert-base-japanese"""
            with self.assertLogs("""transformers"""      ,      level="""WARNING"""  ) as cm:
                  BertTokenizer.from_pretrained(a__  )
                  self.assertTrue(
                      cm.records[0].message.startswith(
                          """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                          """ is called from."""  )  )
            _lowerCAmelCase												=							"""bert-base-cased"""
            with self.assertLogs("""transformers"""      ,      level="""WARNING"""  ) as cm:
                  BertJapaneseTokenizer.from_pretrained(a__  )
                  self.assertTrue(
                      cm.records[0].message.startswith(
                          """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                          """ is called from."""  )  )













'''simple docstring'''






import os
import tempfile
import unittest

from transformers.models.marian.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
from transformers.testing_utils import slow
from transformers.utils import cached_property




@unittest.skipUnless(os.path.exists(snake_case_       )							,		'''Tatoeba directory does not exist.'''       )
class 			UpperCAmelCase					(      unittest.TestCase       ):




      @cached_property
      def     lowercase__   (      self      :			int  )    ->				Any:
            _lowerCAmelCase												=							tempfile.mkdtemp()
            return TatoebaConverter(save_dir=__snake_case  )




      @slow
      def     lowercase__   (      self      :			Dict  )    ->				int:
            self.resolver.convert_models(["""heb-eng"""]  )




      @slow
      def     lowercase__   (      self      :			Optional[int]  )    ->				Tuple:
            _lowerCAmelCase					,							_lowerCAmelCase												=							self.resolver.write_model_card("""opus-mt-he-en"""      ,      dry_run=__snake_case  )
            assert mmeta["long_pair"] == "heb-eng"






