


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 		__a				(    UpperCAmelCase   ,				unittest.TestCase      ):
		_a     :  List[Any]           =     BertJapaneseTokenizer
		_a     :  Tuple           =     False
		_a     :  Dict           =     True
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									super().setUp()

									_UpperCAmelCase					=      [
									    '[UNK]',
									    '[CLS]',
									    '[SEP]',
									    'こんにちは',
									    'こん',
									    'にちは',
									    'ばんは',
									    '##こん',
									    '##にちは',
									    '##ばんは',
									    '世界',
									    '##世界',
									    '、',
									    '##、',
									    '。',
									    '##。',
									]

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['vocab_file']						)
									with open(self.vocab_file      ,   'w'      ,   encoding='utf-8'						) as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	str:




									"""simple docstring"""





									_UpperCAmelCase					=      'こんにちは、世界。 \nこんばんは、世界。'
									_UpperCAmelCase					=      'こんにちは 、 世界 。 こんばんは 、 世界 。'
									return input_text, output_text
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Any:




									"""simple docstring"""





									_UpperCAmelCase		,	_UpperCAmelCase					=      self.get_input_output_texts(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.encode(_SCREAMING_SNAKE_CASE      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.decode(_SCREAMING_SNAKE_CASE      ,   clean_up_tokenization_spaces=_SCREAMING_SNAKE_CASE						)
									return text, ids
		def      UpperCAmelCase__    (      self						)			->	int:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	str:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	Any:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class(self.vocab_file						)

									_UpperCAmelCase					=      tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)
		def      UpperCAmelCase__    (      self						)			->	List[str]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class(self.vocab_file      ,   word_tokenizer_type='mecab'						)
									self.assertIsNotNone(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      'こんにちは、世界。\nこんばんは、世界。'
									_UpperCAmelCase					=      tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   'tokenizer.bin'						)
									with open(_SCREAMING_SNAKE_CASE      ,   'wb'						) as handle:
																pickle.dump(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									with open(_SCREAMING_SNAKE_CASE      ,   'rb'						) as handle:
																_UpperCAmelCase					=      pickle.load(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokenizer_new.tokenize(_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      MecabTokenizer(mecab_dic='ipadic'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									try:
																_UpperCAmelCase					=      MecabTokenizer(mecab_dic='unidic_lite'						)
									except ModuleNotFoundError:
																return

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									try:
																_UpperCAmelCase					=      MecabTokenizer(mecab_dic='unidic'						)
									except ModuleNotFoundError:
																return

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      MecabTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE      ,   mecab_dic='ipadic'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									try:
																_UpperCAmelCase					=      MecabTokenizer(
																    do_lower_case=_SCREAMING_SNAKE_CASE      ,   normalize_text=_SCREAMING_SNAKE_CASE      ,   mecab_option='-d /usr/local/lib/mecab/dic/jumandic'						)
									except RuntimeError:
																# if dict doesn't exist in the system, previous code raises this error.
																return

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									_UpperCAmelCase					=      MecabTokenizer(normalize_text=_SCREAMING_SNAKE_CASE      ,   mecab_dic='ipadic'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']      ,   )
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class(self.vocab_file      ,   word_tokenizer_type='sudachi'						)
									self.assertIsNotNone(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      'こんにちは、世界。\nこんばんは、世界。'
									_UpperCAmelCase					=      tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   'tokenizer.bin'						)
									with open(_SCREAMING_SNAKE_CASE      ,   'wb'						) as handle:
																pickle.dump(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									with open(_SCREAMING_SNAKE_CASE      ,   'rb'						) as handle:
																_UpperCAmelCase					=      pickle.load(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokenizer_new.tokenize(_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(sudachi_dict_type='core'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']      ,   )
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	List[str]:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(sudachi_dict_type='core'      ,   sudachi_split_mode='A'						)

									self.assertListEqual(tokenizer.tokenize('外国人参政権'						)      ,   ['外国', '人', '参政', '権']						)
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	str:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(sudachi_dict_type='core'      ,   sudachi_split_mode='B'						)

									self.assertListEqual(tokenizer.tokenize('外国人参政権'						)      ,   ['外国人', '参政権']						)
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Any:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(sudachi_dict_type='core'      ,   sudachi_split_mode='C'						)

									self.assertListEqual(tokenizer.tokenize('外国人参政権'						)      ,   ['外国人参政権']						)
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE      ,   sudachi_dict_type='core'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']      ,   )
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(normalize_text=_SCREAMING_SNAKE_CASE      ,   sudachi_dict_type='core'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']      ,   )
		@require_sudachi
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      SudachiTokenizer(trim_whitespace=_SCREAMING_SNAKE_CASE      ,   sudachi_dict_type='core'						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']      ,   )
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class(self.vocab_file      ,   word_tokenizer_type='jumanpp'						)
									self.assertIsNotNone(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      'こんにちは、世界。\nこんばんは、世界。'
									_UpperCAmelCase					=      tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   'tokenizer.bin'						)
									with open(_SCREAMING_SNAKE_CASE      ,   'wb'						) as handle:
																pickle.dump(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									with open(_SCREAMING_SNAKE_CASE      ,   'rb'						) as handle:
																_UpperCAmelCase					=      pickle.load(_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokenizer_new.tokenize(_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']      ,   )
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      JumanppTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']      ,   )
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      JumanppTokenizer(normalize_text=_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']      ,   )
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=      JumanppTokenizer(trim_whitespace=_SCREAMING_SNAKE_CASE						)

									self.assertListEqual(
									    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)      ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']      ,   )
		@require_jumanpp
		def      UpperCAmelCase__    (      self						)			->	int:




									"""simple docstring"""





									_UpperCAmelCase					=      JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'						)      ,   ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']      ,   )
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      ['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

									_UpperCAmelCase					=      {}
									for i, token in enumerate(_SCREAMING_SNAKE_CASE						):
																_UpperCAmelCase					=      i
									_UpperCAmelCase					=      WordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE      ,   unk_token='[UNK]'						)

									self.assertListEqual(tokenizer.tokenize(''						)      ,   []						)

									self.assertListEqual(tokenizer.tokenize('こんにちは'						)      ,   ['こんにちは']						)

									self.assertListEqual(tokenizer.tokenize('こんばんは'						)      ,   ['こん', '##ばんは']						)

									self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'						)      ,   ['こん', '##ばんは', '[UNK]', 'こんにちは']						)
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'						)
									_UpperCAmelCase					=      tokenizer.subword_tokenizer

									_UpperCAmelCase					=      subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']						)

									_UpperCAmelCase					=      subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']						)






		def      UpperCAmelCase__    (      self						)			->	Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'						)

									_UpperCAmelCase					=      tokenizer.encode('ありがとう。'      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.encode('どういたしまして。'      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class 		__a				(    UpperCAmelCase   ,				unittest.TestCase      ):
		_a     :  Optional[Any]           =     BertJapaneseTokenizer
		_a     :  str           =     False
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									super().setUp()

									_UpperCAmelCase					=      ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['vocab_file']						)
									with open(self.vocab_file      ,   'w'      ,   encoding='utf-8'						) as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)
		def      UpperCAmelCase__    (      self      ,   **_SCREAMING_SNAKE_CASE						)			->	List[str]:




									"""simple docstring"""





									return BertJapaneseTokenizer.from_pretrained(self.tmpdirname      ,   subword_tokenizer_type='character'      ,   **_SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      'こんにちは、世界。 \nこんばんは、世界。'
									_UpperCAmelCase					=      'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
									return input_text, output_text
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	Tuple:




									"""simple docstring"""





									pass  # TODO add if relevant
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class(self.vocab_file      ,   subword_tokenizer_type='character'						)

									_UpperCAmelCase					=      tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'						)
									self.assertListEqual(
									    _SCREAMING_SNAKE_CASE      ,   ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']						)
									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)
		def      UpperCAmelCase__    (      self						)			->	Any:




									"""simple docstring"""





									_UpperCAmelCase					=      ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

									_UpperCAmelCase					=      {}
									for i, token in enumerate(_SCREAMING_SNAKE_CASE						):
																_UpperCAmelCase					=      i
									_UpperCAmelCase					=      CharacterTokenizer(vocab=_SCREAMING_SNAKE_CASE      ,   unk_token='[UNK]'						)

									self.assertListEqual(tokenizer.tokenize(''						)      ,   []						)

									self.assertListEqual(tokenizer.tokenize('こんにちは'						)      ,   ['こ', 'ん', 'に', 'ち', 'は']						)

									self.assertListEqual(tokenizer.tokenize('こんにちほ'						)      ,   ['こ', 'ん', 'に', 'ち', '[UNK]']						)






		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'						)

									_UpperCAmelCase					=      tokenizer.encode('ありがとう。'      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.encode('どういたしまして。'      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class 		__a				(    unittest.TestCase      ):
		def      UpperCAmelCase__    (      self						)			->	Dict:




									"""simple docstring"""





									_UpperCAmelCase					=      'cl-tohoku/bert-base-japanese'
									_UpperCAmelCase					=      AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE						)
									self.assertIsInstance(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)



class 		__a				(    unittest.TestCase      ):
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      'cl-tohoku/bert-base-japanese'
									with self.assertLogs('transformers'      ,   level='WARNING'						) as cm:
																BertTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE						)
																self.assertTrue(
																    cm.records[0].message.startswith(
																        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
																        ' is called from.'						)						)
									_UpperCAmelCase					=      'bert-base-cased'
									with self.assertLogs('transformers'      ,   level='WARNING'						) as cm:
																BertJapaneseTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE						)
																self.assertTrue(
																    cm.records[0].message.startswith(
																        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
																        ' is called from.'						)						)



from __future__ import annotations

from PIL import Image

# Define glider example
lowerCAmelCase__  :str             =							[
    [0, 1, 0, 0, 0, 0, 0, 0],
    [0, 0, 1, 0, 0, 0, 0, 0],
    [1, 1, 1, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0, 0, 0],
]

# Define blinker example
lowerCAmelCase__  :Any             =							[[0, 1, 0], [0, 1, 0], [0, 1, 0]]

def 	lowerCAmelCase__			(      a__:		list[list[int]]	)  ->		list[list[int]]:


							'''simple docstring'''





							_UpperCAmelCase					=      []
							for i in range(len(a__	)	):
														_UpperCAmelCase					=      []
														for j in range(len(cells[i]	)	):
																					# Get the number of live neighbours
																					_UpperCAmelCase					=      0
																					if i > 0 and j > 0:
																												neighbour_count += cells[i - 1][j - 1]
																					if i > 0:
																												neighbour_count += cells[i - 1][j]
																					if i > 0 and j < len(cells[i]	) - 1:
																												neighbour_count += cells[i - 1][j + 1]
																					if j > 0:
																												neighbour_count += cells[i][j - 1]
																					if j < len(cells[i]	) - 1:
																												neighbour_count += cells[i][j + 1]
																					if i < len(a__	) - 1 and j > 0:
																												neighbour_count += cells[i + 1][j - 1]
																					if i < len(a__	) - 1:
																												neighbour_count += cells[i + 1][j]
																					if i < len(a__	) - 1 and j < len(cells[i]	) - 1:
																												neighbour_count += cells[i + 1][j + 1]

																					# Rules of the game of life (excerpt from Wikipedia):
																					# 1. Any live cell with two or three live neighbours survives.
																					# 2. Any dead cell with three live neighbours becomes a live cell.
																					# 3. All other live cells die in the next generation.
																					#    Similarly, all other dead cells stay dead.
																					_UpperCAmelCase					=      cells[i][j] == 1
																					if (
																					    (alive and 2 <= neighbour_count <= 3)
																					    or not alive
																					    and neighbour_count == 3
																					):
																												next_generation_row.append(1	)
																					else:
																												next_generation_row.append(0	)

														next_generation.append(a__	)
							return next_generation

def 	lowerCAmelCase__			(      a__:		list[list[int]]      ,  a__:		int	)  ->		list[Image.Image]:


							'''simple docstring'''





							_UpperCAmelCase					=      []
							for _ in range(a__	):
														# Create output image
														_UpperCAmelCase					=      Image.new('RGB'      ,  (len(cells[0]	), len(a__	))	)
														_UpperCAmelCase					=      img.load()

														# Save cells to image
														for x in range(len(a__	)	):
																					for y in range(len(cells[0]	)	):
																												_UpperCAmelCase					=      2_5_5 - cells[y][x] * 2_5_5
																												_UpperCAmelCase					=      (colour, colour, colour)

        # Save image
														images.append(a__	)
														_UpperCAmelCase					=      new_generation(a__	)
							return images


if __name__ == "__main__":
		lowerCAmelCase__  :Tuple             =							generate_images(GLIDER, 1_6)
		images[0].save('''out.gif''', save_all=True, append_images=images[1:])
