


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 		__a				(    UpperCAmelCase   ,				unittest.TestCase      ):
		_a     :  List[Any]           =     BertJapaneseTokenizer
		_a     :  Tuple           =     False
		_a     :  Dict           =     True
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									super().setUp()

									_UpperCAmelCase					=      [
									    '[UNK]',
									    '[CLS]',
									    '[SEP]',
									    'こんにちは',
									    'こん',
									    'にちは',
									    'ばんは',
									    '##こん',
									    '##にちは',
									    '##ばんは',
									    '世界',
									    '##世界',
									    '、',
									    '##、',
									    '。',
									    '##。',
									]

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['vocab_file']						)
									with open(self.vocab_file      ,   'w'      ,   encoding='utf-8'						) as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	str:




									"""simple docstring"""





									_UpperCAmelCase					=      'こんにちは、世界。 \nこんばんは、世界。'
									_UpperCAmelCase					=      'こんにちは 、 世界 。 こんばんは 、 世界 。'
									return input_text, output_text
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Any:




									"""simple docstring"""





									_UpperCAmelCase		,	_UpperCAmelCase					=      self.get_input_output_texts(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.encode(_SCREAMING_SNAKE_CASE      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.decode(_SCREAMING_SNAKE_CASE      ,   clean_up_tokenization_spaces=_SCREAMING_SNAKE_CASE						)
									return text, ids
		def      UpperCAmelCase__    (      self						)			->	int:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	str:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	Any:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class(self.vocab_file						)

									_UpperCAmelCase					=      tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)
		def      UpperCAmelCase__    (      self						)			->	List[str]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class(self.vocab_file      ,   word_tokenizer_type='mecab'						)
									self.assertIsNotNone(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      'こんにちは、世界。\nこんばんは、世界。'
									_UpperCAmelCase					=      tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   'tokenizer.bin'						)
									with open(_SCREAMING_SNAKE_CASE      ,   'wb'						) as handle:
																pickle.dump(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									with open(_SCREAMING_SNAKE_CASE      ,   'rb'						) as handle:
																_UpperCAmelCase					=      pickle.load(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokenizer_new.tokenize(_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      MecabTokenizer(mecab_dic='ipadic'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									try:
																_UpperCAmelCase					=      MecabTokenizer(mecab_dic='unidic_lite'						)
									except ModuleNotFoundError:
																return

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									try:
																_UpperCAmelCase					=      MecabTokenizer(mecab_dic='unidic'						)
									except ModuleNotFoundError:
																return

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      MecabTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE      ,   mecab_dic='ipadic'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									try:
																_UpperCAmelCase					=      MecabTokenizer(
																    do_lower_case=_SCREAMING_SNAKE_CASE      ,   normalize_text=_SCREAMING_SNAKE_CASE      ,   mecab_option='-d /usr/local/lib/mecab/dic/jumandic'						)
									except RuntimeError:
																# if dict doesn't exist in the system, previous code raises this error.
																return

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									_UpperCAmelCase					=      MecabTokenizer(normalize_text=_SCREAMING_SNAKE_CASE      ,   mecab_dic='ipadic'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']      ,   )
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class(self.vocab_file      ,   word_tokenizer_type='sudachi'						)
									self.assertIsNotNone(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      'こんにちは、世界。\nこんばんは、世界。'
									_UpperCAmelCase					=      tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   'tokenizer.bin'						)
									with open(_SCREAMING_SNAKE_CASE      ,   'wb'						) as handle:
																pickle.dump(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									with open(_SCREAMING_SNAKE_CASE      ,   'rb'						) as handle:
																_UpperCAmelCase					=      pickle.load(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokenizer_new.tokenize(_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(sudachi_dict_type='core'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']      ,   )
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	List[str]:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(sudachi_dict_type='core'      ,   sudachi_split_mode='A'						)

									self.assertListEqual(tokenizer.tokenize('外国人参政権'						)      ,   ['外国', '人', '参政', '権']						)
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	str:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(sudachi_dict_type='core'      ,   sudachi_split_mode='B'						)

									self.assertListEqual(tokenizer.tokenize('外国人参政権'						)      ,   ['外国人', '参政権']						)
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Any:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(sudachi_dict_type='core'      ,   sudachi_split_mode='C'						)

									self.assertListEqual(tokenizer.tokenize('外国人参政権'						)      ,   ['外国人参政権']						)
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE      ,   sudachi_dict_type='core'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']      ,   )
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(normalize_text=_SCREAMING_SNAKE_CASE      ,   sudachi_dict_type='core'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']      ,   )
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(trim_whitespace=_SCREAMING_SNAKE_CASE      ,   sudachi_dict_type='core'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']      ,   )
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class(self.vocab_file      ,   word_tokenizer_type='jumanpp'						)
									self.assertIsNotNone(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      'こんにちは、世界。\nこんばんは、世界。'
									_UpperCAmelCase					=      tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   'tokenizer.bin'						)
									with open(_SCREAMING_SNAKE_CASE      ,   'wb'						) as handle:
																pickle.dump(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									with open(_SCREAMING_SNAKE_CASE      ,   'rb'						) as handle:
																_UpperCAmelCase					=      pickle.load(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokenizer_new.tokenize(_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']      ,   )
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      JumanppTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']      ,   )
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      JumanppTokenizer(normalize_text=_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']      ,   )
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=      JumanppTokenizer(trim_whitespace=_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']      ,   )
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	int:




									"""simple docstring"""





									_UpperCAmelCase					=      JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'						)      ,   ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      ['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

									_UpperCAmelCase					=      {}
									for i, token in enumerate(_SCREAMING_SNAKE_CASE						):
																_UpperCAmelCase					=      i
									_UpperCAmelCase					=      WordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE      ,   unk_token='[UNK]'						)

									self.assertListEqual(tokenizer.tokenize(''						)      ,   []						)

									self.assertListEqual(tokenizer.tokenize('こんにちは'						)      ,   ['こんにちは']						)

									self.assertListEqual(tokenizer.tokenize('こんばんは'						)      ,   ['こん', '##ばんは']						)

									self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'						)      ,   ['こん', '##ばんは', '[UNK]', 'こんにちは']						)
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'						)
									_UpperCAmelCase					=      tokenizer.subword_tokenizer

									_UpperCAmelCase					=      subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']						)

									_UpperCAmelCase					=      subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']						)






		def      UpperCAmelCase__    (      self						)			->	Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'						)

									_UpperCAmelCase					=      tokenizer.encode('ありがとう。'      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.encode('どういたしまして。'      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class 		__a				(    UpperCAmelCase   ,				unittest.TestCase      ):
		_a     :  Optional[Any]           =     BertJapaneseTokenizer
		_a     :  str           =     False
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									super().setUp()

									_UpperCAmelCase					=      ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['vocab_file']						)
									with open(self.vocab_file      ,   'w'      ,   encoding='utf-8'						) as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)
		def      UpperCAmelCase__    (      self      ,   **_SCREAMING_SNAKE_CASE						)			->	List[str]:




									"""simple docstring"""





									return BertJapaneseTokenizer.from_pretrained(self.tmpdirname      ,   subword_tokenizer_type='character'      ,   **_SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      'こんにちは、世界。 \nこんばんは、世界。'
									_UpperCAmelCase					=      'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
									return input_text, output_text
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	Tuple:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class(self.vocab_file      ,   subword_tokenizer_type='character'						)

									_UpperCAmelCase					=      tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'						)
									self.assertListEqual(
									    _SCREAMING_SNAKE_CASE      ,   ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']						)
									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)
		def      UpperCAmelCase__    (      self						)			->	Any:




									"""simple docstring"""





									_UpperCAmelCase					=      ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

									_UpperCAmelCase					=      {}
									for i, token in enumerate(_SCREAMING_SNAKE_CASE						):
																_UpperCAmelCase					=      i
									_UpperCAmelCase					=      CharacterTokenizer(vocab=_SCREAMING_SNAKE_CASE      ,   unk_token='[UNK]'						)

									self.assertListEqual(tokenizer.tokenize(''						)      ,   []						)

									self.assertListEqual(tokenizer.tokenize('こんにちは'						)      ,   ['こ', 'ん', 'に', 'ち', 'は']						)

									self.assertListEqual(tokenizer.tokenize('こんにちほ'						)      ,   ['こ', 'ん', 'に', 'ち', '[UNK]']						)






		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'						)

									_UpperCAmelCase					=      tokenizer.encode('ありがとう。'      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.encode('どういたしまして。'      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class 		__a				(    unittest.TestCase      ):
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									_UpperCAmelCase					=      'cl-tohoku/bert-base-japanese'
									_UpperCAmelCase					=      AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE						)
									self.assertIsInstance(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)



class 		__a				(    unittest.TestCase      ):
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      'cl-tohoku/bert-base-japanese'
									with self.assertLogs('transformers'      ,   level='WARNING'						) as cm:
																BertTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE						)
																self.assertTrue(
																    cm.records[0].message.startswith(
																        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
																        ' is called from.'						)						)
									_UpperCAmelCase					=      'bert-base-cased'
									with self.assertLogs('transformers'      ,   level='WARNING'						) as cm:
																BertJapaneseTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE						)
																self.assertTrue(
																    cm.records[0].message.startswith(
																        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
																        ' is called from.'						)						)




import math
import random

def 	lowerCAmelCase__			(      a__:		float      ,  a__:		bool = False	)  ->		float:


							'''simple docstring'''





							if deriv:
														return value * (1 - value)
							return 1 / (1 + math.exp(-value	))


# Initial Value
lowerCAmelCase__  :Optional[Any]             =							0.02

def 	lowerCAmelCase__			(      a__:		int      ,  a__:		int	)  ->		float:


							'''simple docstring'''





							_UpperCAmelCase					=      float(2 * (random.randint(1      ,  1_0_0	)) - 1	)

							for _ in range(a__	):
														# Forward propagation
														_UpperCAmelCase					=      sigmoid_function(INITIAL_VALUE * weight	)
														# How much did we miss?
														_UpperCAmelCase					=      (expected / 1_0_0) - layer_a
														# Error delta
														_UpperCAmelCase					=      layer_1_error * sigmoid_function(a__      ,  a__	)
														# Update weight
														weight += INITIAL_VALUE * layer_1_delta

							return layer_a * 1_0_0


if __name__ == "__main__":
		import doctest

		doctest.testmod()

		lowerCAmelCase__  :List[Any]             =							int(input('''Expected value: '''))
		lowerCAmelCase__  :Any             =							int(input('''Number of propagations: '''))
		print(forward_propagation(expected, number_propagations))
