



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 				A_      (					SCREAMING_SNAKE_CASE_     ,       unittest.TestCase    ):



						"""simple docstring"""
						__UpperCamelCase		    =							CodeGenTokenizer
						__UpperCamelCase		    =							CodeGenTokenizerFast
						__UpperCamelCase		    =							True
						__UpperCamelCase		    =							{"""add_prefix_space""": True}
						__UpperCamelCase		    =							False






						def      UpperCAmelCase__					( self			:List[Any] )    ->      str:
												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												UpperCAmelCase					=			[
												    'l',
												    'o',
												    'w',
												    'e',
												    'r',
												    's',
												    't',
												    'i',
												    'd',
												    'n',
												    '\u0120',
												    '\u0120l',
												    '\u0120n',
												    '\u0120lo',
												    '\u0120low',
												    'er',
												    '\u0120lowest',
												    '\u0120newer',
												    '\u0120wider',
												    '<unk>',
												    '<|endoftext|>',
												]
												UpperCAmelCase					=			dict(zip(lowercase_       ,  range(len(lowercase_ ) ) ) )
												UpperCAmelCase					=			['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
												UpperCAmelCase					=			{'unk_token': '<unk>'}

												UpperCAmelCase					=			os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES['vocab_file'] )
												UpperCAmelCase					=			os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES['merges_file'] )
												with open(self.vocab_file       ,  'w'       ,  encoding='utf-8' ) as fp:
																		fp.write(json.dumps(lowercase_ ) + '\n' )
												with open(self.merges_file       ,  'w'       ,  encoding='utf-8' ) as fp:
																		fp.write('\n'.join(lowercase_ ) )






						def      UpperCAmelCase__					( self			:Union[str, Any]       ,  **lowercase_			:List[Any] )    ->      Optional[Any]:
												kwargs.update(self.special_tokens_map )
												return CodeGenTokenizer.from_pretrained(self.tmpdirname       ,  **lowercase_ )






						def      UpperCAmelCase__					( self			:Optional[Any]       ,  **lowercase_			:Dict )    ->      str:
												kwargs.update(self.special_tokens_map )
												return CodeGenTokenizerFast.from_pretrained(self.tmpdirname       ,  **lowercase_ )






						def      UpperCAmelCase__					( self			:str       ,  lowercase_			:Tuple )    ->      str:
												UpperCAmelCase					=			'lower newer'
												UpperCAmelCase					=			'lower newer'
												return input_text, output_text






						def      UpperCAmelCase__					( self			:List[str] )    ->      List[Any]:
												UpperCAmelCase					=			CodeGenTokenizer(self.vocab_file       ,  self.merges_file       ,  **self.special_tokens_map )
												UpperCAmelCase					=			'lower newer'
												UpperCAmelCase					=			['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
												UpperCAmelCase					=			tokenizer.tokenize(lowercase_       ,  add_prefix_space=lowercase_ )
												self.assertListEqual(lowercase_       ,  lowercase_ )

												UpperCAmelCase					=			tokens + [tokenizer.unk_token]
												UpperCAmelCase					=			[14, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_ )       ,  lowercase_ )






						def      UpperCAmelCase__					( self			:Optional[int] )    ->      Any:
												if not self.test_rust_tokenizer:
																		return

												UpperCAmelCase					=			self.get_tokenizer()
												UpperCAmelCase					=			self.get_rust_tokenizer(add_prefix_space=lowercase_ )

												UpperCAmelCase					=			'lower newer'

												# Testing tokenization
												UpperCAmelCase					=			tokenizer.tokenize(lowercase_       ,  add_prefix_space=lowercase_ )
												UpperCAmelCase					=			rust_tokenizer.tokenize(lowercase_ )
												self.assertListEqual(lowercase_       ,  lowercase_ )

												# Testing conversion to ids without special tokens
												UpperCAmelCase					=			tokenizer.encode(lowercase_       ,  add_special_tokens=lowercase_       ,  add_prefix_space=lowercase_ )
												UpperCAmelCase					=			rust_tokenizer.encode(lowercase_       ,  add_special_tokens=lowercase_ )
												self.assertListEqual(lowercase_       ,  lowercase_ )

												# Testing conversion to ids with special tokens
												UpperCAmelCase					=			self.get_rust_tokenizer(add_prefix_space=lowercase_ )
												UpperCAmelCase					=			tokenizer.encode(lowercase_       ,  add_prefix_space=lowercase_ )
												UpperCAmelCase					=			rust_tokenizer.encode(lowercase_ )
												self.assertListEqual(lowercase_       ,  lowercase_ )

												# Testing the unknown token
												UpperCAmelCase					=			tokens + [rust_tokenizer.unk_token]
												UpperCAmelCase					=			[14, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowercase_ )       ,  lowercase_ )






						def      UpperCAmelCase__					( self			:Union[str, Any]       ,  *lowercase_			:Optional[int]       ,  **lowercase_			:int )    ->      Dict:
												# It's very difficult to mix/test pretokenization with byte-level
												# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
												pass






						def      UpperCAmelCase__					( self			:Dict       ,  lowercase_			:Any=15 )    ->      str:
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
																								UpperCAmelCase					=			self.rust_tokenizer_class.from_pretrained(lowercase_       ,  **lowercase_ )

																								# Simple input
																								UpperCAmelCase					=			'This is a simple input'
																								UpperCAmelCase					=			['This is a simple input 1', 'This is a simple input 2']
																								UpperCAmelCase					=			('This is a simple input', 'This is a pair')
																								UpperCAmelCase					=			[
																								    ('This is a simple input 1', 'This is a simple input 2'),
																								    ('This is a simple pair 1', 'This is a simple pair 2'),
																								]

																								# Simple input tests
																								self.assertRaises(lowercase_       ,  tokenizer_r.encode       ,  lowercase_       ,  max_length=lowercase_       ,  padding='max_length' )

																								# Simple input
																								self.assertRaises(lowercase_       ,  tokenizer_r.encode_plus       ,  lowercase_       ,  max_length=lowercase_       ,  padding='max_length' )

																								# Simple input
																								self.assertRaises(
																								    lowercase_       ,  tokenizer_r.batch_encode_plus       ,  lowercase_       ,  max_length=lowercase_       ,  padding='max_length'       ,  )

																								# Pair input
																								self.assertRaises(lowercase_       ,  tokenizer_r.encode       ,  lowercase_       ,  max_length=lowercase_       ,  padding='max_length' )

																								# Pair input
																								self.assertRaises(lowercase_       ,  tokenizer_r.encode_plus       ,  lowercase_       ,  max_length=lowercase_       ,  padding='max_length' )

																								# Pair input
																								self.assertRaises(
																								    lowercase_       ,  tokenizer_r.batch_encode_plus       ,  lowercase_       ,  max_length=lowercase_       ,  padding='max_length'       ,  )






						def      UpperCAmelCase__					( self			:Optional[Any] )    ->      List[str]:
												UpperCAmelCase					=			CodeGenTokenizer.from_pretrained(self.tmpdirname       ,  pad_token='<pad>' )

												# Simple input
												UpperCAmelCase					=			'This is a simple input'
												UpperCAmelCase					=			['This is a simple input looooooooong', 'This is a simple input']
												UpperCAmelCase					=			('This is a simple input', 'This is a pair')
												UpperCAmelCase					=			[
												    ('This is a simple input loooooong', 'This is a simple input'),
												    ('This is a simple pair loooooong', 'This is a simple pair'),
												]

												UpperCAmelCase					=			tokenizer.pad_token_id

												UpperCAmelCase					=			tokenizer(lowercase_       ,  padding='max_length'       ,  max_length=30       ,  return_tensors='np' )
												UpperCAmelCase					=			tokenizer(lowercase_       ,  padding=lowercase_       ,  truncate=lowercase_       ,  return_tensors='np' )
												UpperCAmelCase					=			tokenizer(*lowercase_       ,  padding='max_length'       ,  max_length=60       ,  return_tensors='np' )
												UpperCAmelCase					=			tokenizer(lowercase_       ,  padding=lowercase_       ,  truncate=lowercase_       ,  return_tensors='np' )

												# s
												# test single string max_length padding
												self.assertEqual(out_s['input_ids'].shape[-1]       ,  30 )
												self.assertTrue(pad_token_id in out_s['input_ids'] )
												self.assertTrue(0 in out_s['attention_mask'] )

												# s2
												# test automatic padding
												self.assertEqual(out_sa['input_ids'].shape[-1]       ,  33 )
												# long slice doesn't have padding
												self.assertFalse(pad_token_id in out_sa['input_ids'][0] )
												self.assertFalse(0 in out_sa['attention_mask'][0] )
												# short slice does have padding
												self.assertTrue(pad_token_id in out_sa['input_ids'][1] )
												self.assertTrue(0 in out_sa['attention_mask'][1] )

												# p
												# test single pair max_length padding
												self.assertEqual(out_p['input_ids'].shape[-1]       ,  60 )
												self.assertTrue(pad_token_id in out_p['input_ids'] )
												self.assertTrue(0 in out_p['attention_mask'] )

												# p2
												# test automatic padding pair
												self.assertEqual(out_pa['input_ids'].shape[-1]       ,  52 )
												# long slice pair doesn't have padding
												self.assertFalse(pad_token_id in out_pa['input_ids'][0] )
												self.assertFalse(0 in out_pa['attention_mask'][0] )
												# short slice pair does have padding
												self.assertTrue(pad_token_id in out_pa['input_ids'][1] )
												self.assertTrue(0 in out_pa['attention_mask'][1] )






						def      UpperCAmelCase__					( self			:int )    ->      List[Any]:
												UpperCAmelCase					=			'$$$'
												UpperCAmelCase					=			CodeGenTokenizer.from_pretrained(self.tmpdirname       ,  bos_token=lowercase_       ,  add_bos_token=lowercase_ )

												UpperCAmelCase					=			'This is a simple input'
												UpperCAmelCase					=			['This is a simple input 1', 'This is a simple input 2']

												UpperCAmelCase					=			tokenizer.bos_token_id

												UpperCAmelCase					=			tokenizer(lowercase_ )
												UpperCAmelCase					=			tokenizer(lowercase_ )

												self.assertEqual(out_s.input_ids[0]       ,  lowercase_ )
												self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids ) )

												UpperCAmelCase					=			tokenizer.decode(out_s.input_ids )
												UpperCAmelCase					=			tokenizer.batch_decode(out_sa.input_ids )

												self.assertEqual(decode_s.split()[0]       ,  lowercase_ )
												self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa ) )






						@slow
						def      UpperCAmelCase__					( self			:int )    ->      Union[str, Any]:
												UpperCAmelCase					=			CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono' )

												UpperCAmelCase					=			'\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
												UpperCAmelCase					=			'\nif len_a > len_b:      result = a\nelse:      result = b'

												UpperCAmelCase					=			tokenizer.encode(lowercase_ )
												UpperCAmelCase					=			['^#', re.escape('<|endoftext|>' ), '^\'\'\'', '^"""', '\n\n\n']
												UpperCAmelCase					=			tokenizer.decode(lowercase_       ,  truncate_before_pattern=lowercase_ )
												self.assertEqual(lowercase_       ,  lowercase_ )






						def      UpperCAmelCase__					( self			:Union[str, Any] )    ->      Union[str, Any]:
												pass










"""simple docstring"""


def 			_lowerCAmelCase				(       ):
						for n in range(1		,     1000000						):
												yield n * (n + 1) // 2

def 			_lowerCAmelCase				(       lowercase_						):
						UpperCAmelCase					=			1
						UpperCAmelCase					=			2
						while i * i <= n:
												UpperCAmelCase					=			0
												while n % i == 0:
																		n //= i
																		multiplicity += 1
												divisors_count *= multiplicity + 1
												i += 1
						if n > 1:
												divisors_count *= 2
						return divisors_count

def 			_lowerCAmelCase				(       ):
						return next(i for i in triangle_number_generator() if count_divisors(lowercase_						) > 500						)


if __name__ == "__main__":
				print(solution())





