



import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      UpperCamelCase    (						_UpperCAmelCase       ,    unittest.TestCase     ):
	lowerCAmelCase	:   List[str]				=	MobileBertTokenizer
	lowerCAmelCase	:   List[Any]				=	MobileBertTokenizerFast
	lowerCAmelCase	:   Optional[int]				=	True
	lowerCAmelCase	:   Optional[int]				=	True
	lowerCAmelCase	:   str				=	filter_non_english
	lowerCAmelCase	:   int				=	"""google/mobilebert-uncased"""



	def 			__A							(       self		):
							super().setUp()

							A__							=       [
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							A__							=       os.path.join(self.tmpdirname			,	VOCAB_FILES_NAMES["vocab_file"]		)
							with open(self.vocab_file			,	"w"			,	encoding="utf-8"		) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)

							A__							=       [
							    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
							    for tokenizer_def in self.tokenizers_list
							]



	def 			__A							(       self			,	UpperCAmelCase__		):
							A__							=       "UNwant\u00E9d,running"
							A__							=       "unwanted, running"
							return input_text, output_text



	def 			__A							(       self		):
							A__							=       self.tokenizer_class(self.vocab_file		)

							A__							=       tokenizer.tokenize("UNwant\u00E9d,running"		)
							self.assertListEqual(UpperCAmelCase__			,	["un", "##want", "##ed", ",", "runn", "##ing"]		)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__		)			,	[9, 6, 7, 12, 10, 11]		)



	def 			__A							(       self		):
							if not self.test_rust_tokenizer:
													return

							A__							=       self.get_tokenizer()
							A__							=       self.get_rust_tokenizer()

							A__							=       "UNwant\u00E9d,running"

							A__							=       tokenizer.tokenize(UpperCAmelCase__		)
							A__							=       rust_tokenizer.tokenize(UpperCAmelCase__		)
							self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)

							A__							=       tokenizer.encode(UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__		)
							A__							=       rust_tokenizer.encode(UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__		)
							self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)

							A__							=       self.get_rust_tokenizer()
							A__							=       tokenizer.encode(UpperCAmelCase__		)
							A__							=       rust_tokenizer.encode(UpperCAmelCase__		)
							self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)

							# With lower casing
							A__							=       self.get_tokenizer(do_lower_case=UpperCAmelCase__		)
							A__							=       self.get_rust_tokenizer(do_lower_case=UpperCAmelCase__		)

							A__							=       "UNwant\u00E9d,running"

							A__							=       tokenizer.tokenize(UpperCAmelCase__		)
							A__							=       rust_tokenizer.tokenize(UpperCAmelCase__		)
							self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)

							A__							=       tokenizer.encode(UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__		)
							A__							=       rust_tokenizer.encode(UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__		)
							self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)

							A__							=       self.get_rust_tokenizer()
							A__							=       tokenizer.encode(UpperCAmelCase__		)
							A__							=       rust_tokenizer.encode(UpperCAmelCase__		)
							self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)



	def 			__A							(       self		):
							A__							=       BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)			,	["ah", "\u535A", "\u63A8", "zz"]		)



	def 			__A							(       self		):
							A__							=       BasicTokenizer(do_lower_case=UpperCAmelCase__		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)			,	["hello", "!", "how", "are", "you", "?"]		)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)			,	["hello"]		)



	def 			__A							(       self		):
							A__							=       BasicTokenizer(do_lower_case=UpperCAmelCase__			,	strip_accents=UpperCAmelCase__		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)			,	["hällo", "!", "how", "are", "you", "?"]		)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)			,	["h\u00E9llo"]		)



	def 			__A							(       self		):
							A__							=       BasicTokenizer(do_lower_case=UpperCAmelCase__			,	strip_accents=UpperCAmelCase__		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)			,	["hallo", "!", "how", "are", "you", "?"]		)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)			,	["hello"]		)



	def 			__A							(       self		):
							A__							=       BasicTokenizer(do_lower_case=UpperCAmelCase__		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)			,	["hallo", "!", "how", "are", "you", "?"]		)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)			,	["hello"]		)



	def 			__A							(       self		):
							A__							=       BasicTokenizer(do_lower_case=UpperCAmelCase__		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)			,	["HeLLo", "!", "how", "Are", "yoU", "?"]		)



	def 			__A							(       self		):
							A__							=       BasicTokenizer(do_lower_case=UpperCAmelCase__			,	strip_accents=UpperCAmelCase__		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)			,	["HäLLo", "!", "how", "Are", "yoU", "?"]		)



	def 			__A							(       self		):
							A__							=       BasicTokenizer(do_lower_case=UpperCAmelCase__			,	strip_accents=UpperCAmelCase__		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)			,	["HaLLo", "!", "how", "Are", "yoU", "?"]		)



	def 			__A							(       self		):
							A__							=       BasicTokenizer(do_lower_case=UpperCAmelCase__			,	never_split=["[UNK]"]		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)			,	["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)



	def 			__A							(       self		):
							A__							=       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							A__							=       {}
							for i, token in enumerate(UpperCAmelCase__		):
													A__							=       i
							A__							=       WordpieceTokenizer(vocab=UpperCAmelCase__			,	unk_token="[UNK]"		)

							self.assertListEqual(tokenizer.tokenize(""		)			,	[]		)

							self.assertListEqual(tokenizer.tokenize("unwanted running"		)			,	["un", "##want", "##ed", "runn", "##ing"]		)

							self.assertListEqual(tokenizer.tokenize("unwantedX running"		)			,	["[UNK]", "runn", "##ing"]		)



	def 			__A							(       self		):
							self.assertTrue(_is_whitespace(" "		)		)
							self.assertTrue(_is_whitespace("\t"		)		)
							self.assertTrue(_is_whitespace("\r"		)		)
							self.assertTrue(_is_whitespace("\n"		)		)
							self.assertTrue(_is_whitespace("\u00A0"		)		)

							self.assertFalse(_is_whitespace("A"		)		)
							self.assertFalse(_is_whitespace("-"		)		)



	def 			__A							(       self		):
							self.assertTrue(_is_control("\u0005"		)		)

							self.assertFalse(_is_control("A"		)		)
							self.assertFalse(_is_control(" "		)		)
							self.assertFalse(_is_control("\t"		)		)
							self.assertFalse(_is_control("\r"		)		)



	def 			__A							(       self		):
							self.assertTrue(_is_punctuation("-"		)		)
							self.assertTrue(_is_punctuation("$"		)		)
							self.assertTrue(_is_punctuation("`"		)		)
							self.assertTrue(_is_punctuation("."		)		)

							self.assertFalse(_is_punctuation("A"		)		)
							self.assertFalse(_is_punctuation(" "		)		)



	def 			__A							(       self		):
							A__							=       self.get_tokenizer()
							A__							=       self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(UpperCAmelCase__		) for t in ["Test", "\xad", "test"]]			,	[["[UNK]"], [], ["[UNK]"]]		)

							self.assertListEqual(
							    [rust_tokenizer.tokenize(UpperCAmelCase__		) for t in ["Test", "\xad", "test"]]			,	[["[UNK]"], [], ["[UNK]"]]		)



	@slow
	def 			__A							(       self		):
							A__							=       self.tokenizer_class.from_pretrained("google/mobilebert-uncased"		)

							A__							=       tokenizer.encode("sequence builders"			,	add_special_tokens=UpperCAmelCase__		)
							A__							=       tokenizer.encode("multi-sequence build"			,	add_special_tokens=UpperCAmelCase__		)

							A__							=       tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__		)
							A__							=       tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__			,	UpperCAmelCase__		)

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]



	def 			__A							(       self		):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
																			A__							=       self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__			,	**UpperCAmelCase__		)

																			A__							=       F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																			A__							=       tokenizer_r.encode_plus(
																			    UpperCAmelCase__			,	return_attention_mask=UpperCAmelCase__			,	return_token_type_ids=UpperCAmelCase__			,	return_offsets_mapping=UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__			,	)

																			A__							=       tokenizer_r.do_lower_case if hasattr(UpperCAmelCase__			,	"do_lower_case"		) else False
																			A__							=       (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), "A"),
																			        ((1, 2), ","),
																			        ((3, 5), "na"),
																			        ((5, 6), "##ï"),
																			        ((6, 8), "##ve"),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), "Allen"),
																			        ((21, 23), "##NL"),
																			        ((23, 24), "##P"),
																			        ((25, 33), "sentence"),
																			        ((33, 34), "."),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), "a"),
																			        ((1, 2), ","),
																			        ((3, 8), "naive"),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), "allen"),
																			        ((21, 23), "##nl"),
																			        ((23, 24), "##p"),
																			        ((25, 33), "sentence"),
																			        ((33, 34), "."),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]			,	tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
																			self.assertEqual([e[0] for e in expected_results]			,	tokens["offset_mapping"]		)



	def 			__A							(       self		):
							A__							=       ["的", "人", "有"]
							A__							=       "".join(UpperCAmelCase__		)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
																			A__							=       True
																			A__							=       self.tokenizer_class.from_pretrained(UpperCAmelCase__			,	**UpperCAmelCase__		)
																			A__							=       self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__			,	**UpperCAmelCase__		)

																			A__							=       tokenizer_p.encode(UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__		)
																			A__							=       tokenizer_r.encode(UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__		)

																			A__							=       tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__		)
																			A__							=       tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__		)

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)
																			self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)

																			A__							=       False
																			A__							=       self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__			,	**UpperCAmelCase__		)
																			A__							=       self.tokenizer_class.from_pretrained(UpperCAmelCase__			,	**UpperCAmelCase__		)

																			A__							=       tokenizer_r.encode(UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__		)
																			A__							=       tokenizer_p.encode(UpperCAmelCase__			,	add_special_tokens=UpperCAmelCase__		)

																			A__							=       tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__		)
																			A__							=       tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__		)

																			# it is expected that only the first Chinese character is not preceded by "##".
																			A__							=       [
																			    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase__		)
																			]
																			self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)
																			self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)










import os
import zipfile

import requests
from get_ci_error_statistics import download_artifact, get_artifacts_links
def  UpperCamelCase						(       _A			:				List[Any]			,							_A			:				List[str]=7    )->       Optional[Any]:





						"""simple docstring"""





						A__							=       None
						if token is not None:
												A__							=       {"Accept": "application/vnd.github+json", "Authorization": f"""Bearer {token}"""}

						# The id of a workflow (not of a workflow run)
						A__							=       "636036"

						A__							=       f"""https://api.github.com/repos/huggingface/transformers/actions/workflows/{workflow_id}/runs"""
						# On `main` branch + event being `schedule` + not returning PRs + only `num_runs` results
						url += f"""?branch=main&event=schedule&exclude_pull_requests=true&per_page={num_runs}"""

						A__							=       requests.get(_A			,							headers=_A    ).json()

						return result["workflow_runs"]
def  UpperCamelCase						(       _A			:				str    )->       Dict:





						"""simple docstring"""





						A__							=       get_daily_ci_runs(_A    )
						A__							=       None
						for workflow_run in workflow_runs:
												if workflow_run["status"] == "completed":
																		A__							=       workflow_run["id"]
																		break

						return workflow_run_id
def  UpperCamelCase						(       _A			:				int			,							_A			:				List[str]			,							_A			:				str    )->       Any:





						"""simple docstring"""





						A__							=       get_last_daily_ci_runs(_A    )
						if workflow_run_id is not None:
												A__							=       get_artifacts_links(worflow_run_id=_A			,							token=_A    )
												for artifact_name in artifact_names:
																		if artifact_name in artifacts_links:
																								A__							=       artifacts_links[artifact_name]
																								download_artifact(
																								    artifact_name=_A			,							artifact_url=_A			,							output_dir=_A			,							token=_A    )
def  UpperCamelCase						(       _A			:				Optional[Any]			,							_A			:				Any			,							_A			:				List[Any]    )->       Optional[int]:





						"""simple docstring"""





						get_last_daily_ci_artifacts(_A			,							_A			,							_A    )

						A__							=       {}
						for artifact_name in artifact_names:
												A__							=       os.path.join(_A			,							f"""{artifact_name}.zip"""    )
												if os.path.isfile(_A    ):
																		A__							=       {}
																		with zipfile.ZipFile(_A    ) as z:
																								for filename in z.namelist():
																														if not os.path.isdir(_A    ):
																																				# read the file
																																				with z.open(_A    ) as f:
																																										A__							=       f.read().decode("UTF-8"    )

						return results





