






import os

import jsonlines
import numpy as np
from tqdm import tqdm


snake_case_           =      2048
snake_case_           =      4096
snake_case_           =      42
snake_case_           =      os.environ.pop('PROCESS_TRAIN', 'false')
snake_case_           =      {'null': 0, 'short': 1, 'long': 2, 'yes': 3, 'no': 4}






def lowerCamelCase__   (							snake_case_     :  Dict	)				->    Tuple:


   def choose_first(snake_case_     :  str	,				snake_case_     :  Optional[Any]=False	):
      assert isinstance(lowerCAmelCase__	,				lowerCAmelCase__	)
      if len(lowerCAmelCase__	) == 1:
         __snake_case								=       answer[0]
         return {k: [answer[k]] for k in answer} if is_long_answer else answer
      for a in answer:
         if is_long_answer:
            __snake_case								=       {k: [a[k]] for k in a}
         if len(a['''start_token''']	) > 0:
            break
      return a

   __snake_case								=       {"""id""": example["""id"""]}
   __snake_case								=       example["""annotations"""]
   __snake_case								=       annotation["""yes_no_answer"""]
   if 0 in yes_no_answer or 1 in yes_no_answer:
      __snake_case								=       ["""yes"""] if 1 in yes_no_answer else ["""no"""]
      __snake_case								=       []
      __snake_case								=       []
      __snake_case								=       ["""<cls>"""]
   else:
      __snake_case								=       ["""short"""]
      __snake_case								=       choose_first(annotation['''short_answers''']	)
      if len(out['''start_token''']	) == 0:
         # answer will be long if short is not available
         __snake_case								=       ["""long"""]
         __snake_case								=       choose_first(annotation['''long_answer''']	,				is_long_answer=lowerCAmelCase__	)
         __snake_case								=       []
      answer.update(lowerCAmelCase__	)

   # disregard some samples
   if len(answer['''start_token''']	) > 1 or answer["start_token"] == answer["end_token"]:
      __snake_case								=       True
   else:
      __snake_case								=       False

   __snake_case								=       ["""start_token""", """end_token""", """start_byte""", """end_byte""", """text"""]
   if not all(isinstance(answer[k]	,				lowerCAmelCase__	) for k in cols	):
      raise ValueError('''Issue in ID'''	,				example['''id''']	)

   return answer






def lowerCamelCase__   (							snake_case_     :  Optional[int]	,				snake_case_     :  List[str]=False	)				->    Optional[int]:
   __snake_case								=       _get_single_answer(lowerCAmelCase__	)
   # bytes are of no use
   del answer["start_byte"]
   del answer["end_byte"]

   # handle yes_no answers explicitly
   if answer["category"][0] in ["yes", "no"]:  # category is list with one element
      __snake_case								=       example["""document"""]["""tokens"""]
      __snake_case								=       []
      for i in range(len(doc['''token''']	)	):
         if not doc["is_html"][i]:
            context.append(doc['''token'''][i]	)
      return {
          "context": " ".join(lowerCAmelCase__	),
          "answer": {
              "start_token": -100,  # ignore index in cross-entropy
              "end_token": -100,  # ignore index in cross-entropy
              "category": answer["category"],
              "span": answer["category"],  # extra
          },
      }

   # later, help in removing all no answers
   if answer["start_token"] == [-1]:
      return {
          "context": "None",
          "answer": {
              "start_token": -1,
              "end_token": -1,
              "category": "null",
              "span": "None",  # extra
          },
      }

   # handling normal samples

   __snake_case								=       ["""start_token""", """end_token"""]
   answer.update({k: answer[k][0] if len(answer[k]	) > 0 else answer[k] for k in cols}	)  # e.g. [10] == 10

   __snake_case								=       example["""document"""]["""tokens"""]
   __snake_case								=       answer["""start_token"""]
   __snake_case								=       answer["""end_token"""]

   __snake_case								=       []
   for i in range(len(doc['''token''']	)	):
      if not doc["is_html"][i]:
         context.append(doc['''token'''][i]	)
      else:
         if answer["start_token"] > i:
            start_token -= 1
         if answer["end_token"] > i:
            end_token -= 1
   __snake_case								=       """ """.join(context[start_token:end_token]	)

   # checking above code
   if assertion:
      __snake_case								=       doc["""is_html"""][answer["""start_token"""] : answer["""end_token"""]]
      __snake_case								=       doc["""token"""][answer["""start_token"""] : answer["""end_token"""]]
      __snake_case								=       """ """.join([old[i] for i in range(len(lowerCAmelCase__	)	) if not is_html[i]]	)
      if new != old:
         print('''ID:'''	,				example['''id''']	)
         print('''New:'''	,				lowerCAmelCase__	,				end='''\n'''	)
         print('''Old:'''	,				lowerCAmelCase__	,				end='''\n\n'''	)

   return {
       "context": " ".join(lowerCAmelCase__	),
       "answer": {
           "start_token": start_token,
           "end_token": end_token - 1,  # this makes it inclusive
           "category": answer["category"],  # either long or short
           "span": new,  # extra
       },
   }






def lowerCamelCase__   (							snake_case_     :  str	,				snake_case_     :  str	,				snake_case_     :  Union[str, Any]=2048	,				snake_case_     :  Union[str, Any]=4096	,				snake_case_     :  int=True	)				->    Tuple:
   __snake_case								=       get_context_and_ans(lowerCAmelCase__	,				assertion=lowerCAmelCase__	)
   __snake_case								=       out["""answer"""]

   # later, removing these samples
   if answer["start_token"] == -1:
      return {
          "example_id": example["id"],
          "input_ids": [[-1]],
          "labels": {
              "start_token": [-1],
              "end_token": [-1],
              "category": ["null"],
          },
      }

   __snake_case								=       tokenizer(example['''question''']['''text''']	,				out['''context''']	).input_ids
   __snake_case								=       input_ids.index(tokenizer.sep_token_id	) + 1

   # return yes/no
   if answer["category"][0] in ["yes", "no"]:  # category is list with one element
      __snake_case								=       []
      __snake_case								=       []
      __snake_case								=       input_ids[:q_len]
      __snake_case								=       range(lowerCAmelCase__	,				len(lowerCAmelCase__	)	,				max_length - doc_stride	)
      for i in doc_start_indices:
         __snake_case								=       i + max_length - q_len
         __snake_case								=       input_ids[i:end_index]
         inputs.append(q_indices + slice	)
         category.append(answer['''category'''][0]	)
         if slice[-1] == tokenizer.sep_token_id:
            break

      return {
          "example_id": example["id"],
          "input_ids": inputs,
          "labels": {
              "start_token": [-100] * len(lowerCAmelCase__	),
              "end_token": [-100] * len(lowerCAmelCase__	),
              "category": category,
          },
      }

   __snake_case								=       out["""context"""].split()
   __snake_case								=       splitted_context[answer["""end_token"""]]
   __snake_case								=       len(
       tokenizer(
           ''' '''.join(splitted_context[: answer['''start_token''']]	)	,				add_special_tokens=lowerCAmelCase__	,				).input_ids	)
   __snake_case								=       len(
       tokenizer(''' '''.join(splitted_context[: answer['''end_token''']]	)	,				add_special_tokens=lowerCAmelCase__	).input_ids	)

   answer["start_token"] += q_len
   answer["end_token"] += q_len

   # fixing end token
   __snake_case								=       len(tokenizer(lowerCAmelCase__	,				add_special_tokens=lowerCAmelCase__	).input_ids	)
   if num_sub_tokens > 1:
      answer["end_token"] += num_sub_tokens - 1

   __snake_case								=       input_ids[answer["""start_token"""] : answer["""end_token"""] + 1]  # right & left are inclusive
   __snake_case								=       answer["""start_token"""]
   __snake_case								=       answer["""end_token"""]

   if assertion:
      __snake_case								=       tokenizer.decode(lowerCAmelCase__	)
      if answer["span"] != new:
         print('''ISSUE IN TOKENIZATION'''	)
         print('''OLD:'''	,				answer['''span''']	)
         print('''NEW:'''	,				lowerCAmelCase__	,				end='''\n\n'''	)

   if len(lowerCAmelCase__	) <= max_length:
      return {
          "example_id": example["id"],
          "input_ids": [input_ids],
          "labels": {
              "start_token": [answer["start_token"]],
              "end_token": [answer["end_token"]],
              "category": answer["category"],
          },
      }

   __snake_case								=       input_ids[:q_len]
   __snake_case								=       range(lowerCAmelCase__	,				len(lowerCAmelCase__	)	,				max_length - doc_stride	)

   __snake_case								=       []
   __snake_case								=       []
   __snake_case								=       []
   __snake_case								=       []  # null, yes, no, long, short
   for i in doc_start_indices:
      __snake_case								=       i + max_length - q_len
      __snake_case								=       input_ids[i:end_index]
      inputs.append(q_indices + slice	)
      assert len(inputs[-1]	) <= max_length, "Issue in truncating length"

      if start_token >= i and end_token <= end_index - 1:
         __snake_case								=       start_token - i + q_len
         __snake_case								=       end_token - i + q_len
         answers_category.append(answer['''category'''][0]	)  # ["short"] -> "short"
      else:
         __snake_case								=       -100
         __snake_case								=       -100
         answers_category.append('''null'''	)
      __snake_case								=       inputs[-1][start_token : end_token + 1]

      answers_start_token.append(lowerCAmelCase__	)
      answers_end_token.append(lowerCAmelCase__	)
      if assertion:
         if new != old and new != [tokenizer.cls_token_id]:
            print('''ISSUE in strided for ID:'''	,				example['''id''']	)
            print('''New:'''	,				tokenizer.decode(lowerCAmelCase__	)	)
            print('''Old:'''	,				tokenizer.decode(lowerCAmelCase__	)	,				end='''\n\n'''	)
      if slice[-1] == tokenizer.sep_token_id:
         break

   return {
       "example_id": example["id"],
       "input_ids": inputs,
       "labels": {
           "start_token": answers_start_token,
           "end_token": answers_end_token,
           "category": answers_category,
       },
   }






def lowerCamelCase__   (							snake_case_     :  int	,				snake_case_     :  Optional[int]	,				snake_case_     :  Union[str, Any]=2048	,				snake_case_     :  str=4096	,				snake_case_     :  str=False	)				->    List[str]:
   __snake_case								=       get_strided_contexts_and_ans(
       lowerCAmelCase__	,				lowerCAmelCase__	,				doc_stride=lowerCAmelCase__	,				max_length=lowerCAmelCase__	,				assertion=lowerCAmelCase__	,				)

   return example






def lowerCamelCase__   (							snake_case_     :  List[str]	,				snake_case_     :  Any	)				->    Tuple:


   with jsonlines.open(lowerCAmelCase__	,				'''a'''	) as writer:
      for example in tqdm(lowerCAmelCase__	,				total=len(lowerCAmelCase__	)	,				desc='''Saving samples ... '''	):
         __snake_case								=       example["""labels"""]
         for ids, start, end, cat in zip(
             example['''input_ids''']	,				labels['''start_token''']	,				labels['''end_token''']	,				labels['''category''']	,				):
            if start == -1 and end == -1:
               continue  # leave waste samples with no answer
            if cat == "null" and np.random.rand() < 0.6:
               continue  # removing 50 % samples
            writer.write(
                {
                    '''input_ids''': ids,
                    '''start_token''': start,
                    '''end_token''': end,
                    '''category''': CATEGORY_MAPPING[cat],
                }	)


if __name__ == "__main__":
    from datasets import load_dataset

    from transformers import BigBirdTokenizer

    snake_case_           =      load_dataset('natural_questions')
    snake_case_           =      BigBirdTokenizer.from_pretrained('google/bigbird-roberta-base')

    snake_case_           =      data['train' if PROCESS_TRAIN == 'true' else 'validation']

    snake_case_           =      {
        'tokenizer': tokenizer,
        'doc_stride': DOC_STRIDE,
        'max_length': MAX_LENGTH,
        'assertion': False,
    }
    snake_case_           =      data.map(prepare_inputs, fn_kwargs=fn_kwargs)
    snake_case_           =      data.remove_columns(['annotations', 'document', 'id', 'question'])
    print(data)

    np.random.seed(SEED)
    snake_case_           =      'nq-training.jsonl' if PROCESS_TRAIN == 'true' else 'nq-validation.jsonl'
    save_to_disk(data, file_name=cache_file_name)








from __future__ import annotations

snake_case_           =      [True] * 1000001
snake_case_           =      2
while i * i <= 1000000:
    if seive[i]:
        for j in range(i * i, 1000001, i):
            snake_case_           =      False
    i += 1






def lowerCamelCase__   (							snake_case_     :  int	)				->    bool:
   return seive[n]






def lowerCamelCase__   (							snake_case_     :  int	)				->    bool:
   return any(digit in '''02468''' for digit in str(snake_case_	)	)






def lowerCamelCase__   (							snake_case_     :  int = 100_0000	)				->    list[int]:
   __snake_case								=       [2]  # result already includes the number 2.
   for num in range(3	,				limit + 1	,				2	):
      if is_prime(snake_case_	) and not contains_an_even_digit(snake_case_	):
         __snake_case								=       str(snake_case_	)
         __snake_case								=       [int(str_num[j:] + str_num[:j]	) for j in range(len(snake_case_	)	)]
         if all(is_prime(snake_case_	) for i in list_nums	):
            result.append(snake_case_	)
   return result






def lowerCamelCase__   (							)				->    int:
   return len(find_circular_primes()	)


if __name__ == "__main__":
    print(F'{len(find_circular_primes()) = }')

