




'''simple docstring'''


import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_snake_case				:				str       							=		logging.get_logger(__name__)

_snake_case				:				Tuple       							=		{'vocab_file': 'spiece.model'}

_snake_case				:				Optional[int]       							=		{
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

_snake_case				:				Tuple       							=		{
    'google/bigbird-roberta-base': 4096,
    'google/bigbird-roberta-large': 4096,
    'google/bigbird-base-trivia-itc': 4096,
}

class       A   (		_a   ):
							lowercase_          =    VOCAB_FILES_NAMES
							lowercase_          =    PRETRAINED_VOCAB_FILES_MAP
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_          =    ['input_ids', 'attention_mask']
							lowercase_          =    []







							def __init__(  self	:  Optional[Any]							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[int]="<unk>"							,      lowerCAmelCase_	:  Tuple="<s>"							,      lowerCAmelCase_	:  List[str]="</s>"							,      lowerCAmelCase_	:  int="<pad>"							,      lowerCAmelCase_	:  Union[str, Any]="[SEP]"							,      lowerCAmelCase_	:  Dict="[MASK]"							,      lowerCAmelCase_	:  Optional[int]="[CLS]"							,      lowerCAmelCase_	:  Optional[Dict[str, Any]] = None							,      **lowerCAmelCase_	:  Union[str, Any]							,      )      ->							None:


														"""simple docstring"""


														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else bos_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else eos_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else unk_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else pad_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else cls_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else sep_token

														# Mask token behave like a normal word, i.e. include the space before it
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else mask_token

														_a     					=    {} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    bos_token=lowerCAmelCase_							,      eos_token=lowerCAmelCase_							,      unk_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      sp_model_kwargs=self.sp_model_kwargs							,      **lowerCAmelCase_							,      )

														_a     					=    vocab_file

														_a     					=    spm.SentencePieceProcessor(**self.sp_model_kwargs )
														self.sp_model.Load(lowerCAmelCase_ )







							@property
							def        __lowerCAmelCase      (  self	:  Union[str, Any] )      ->							int:


														"""simple docstring"""


														return self.sp_model.get_piece_size()







							def        __lowerCAmelCase      (  self	:  str )      ->							List[str]:


														"""simple docstring"""


														_a     					=    {self.convert_ids_to_tokens(lowerCAmelCase_ ): i for i in range(self.vocab_size )}
														vocab.update(self.added_tokens_encoder )
														return vocab







							def __getstate__(  self	:  str )      ->							Tuple:


														"""simple docstring"""


														_a     					=    self.__dict__.copy()
														_a     					=    None
														return state







							def __setstate__(  self	:  List[str]							,      lowerCAmelCase_	:  Any )      ->							Dict:


														"""simple docstring"""


														_a     					=    d

														# for backward compatibility
														if not hasattr(self							,      '''sp_model_kwargs''' ):
																					_a     					=    {}

														_a     					=    spm.SentencePieceProcessor(**self.sp_model_kwargs )
														self.sp_model.Load(self.vocab_file )







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      lowerCAmelCase_	:  str )      ->							List[str]:


														"""simple docstring"""


														return self.sp_model.encode(lowerCAmelCase_							,      out_type=lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  List[str] )      ->							int:


														"""simple docstring"""


														return self.sp_model.piece_to_id(lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  Tuple )      ->							str:


														"""simple docstring"""


														_a     					=    self.sp_model.IdToPiece(lowerCAmelCase_ )
														return token







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  Union[str, Any] )      ->							Optional[Any]:


														"""simple docstring"""


														_a     					=    []
														_a     					=    ''''''
														_a     					=    False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												if not prev_is_special:
																																			out_string += " "
																												out_string += self.sp_model.decode(lowerCAmelCase_ ) + token
																												_a     					=    True
																												_a     					=    []
																					else:
																												current_sub_tokens.append(lowerCAmelCase_ )
																												_a     					=    False
														out_string += self.sp_model.decode(lowerCAmelCase_ )
														return out_string.strip()







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  bool = False							,      lowerCAmelCase_	:  bool = None							,      lowerCAmelCase_	:  bool = True							,      **lowerCAmelCase_	:  Tuple							,      )      ->							str:


														"""simple docstring"""


														_a     					=    kwargs.pop('''use_source_tokenizer'''							,      lowerCAmelCase_ )

														_a     					=    self.convert_ids_to_tokens(lowerCAmelCase_							,      skip_special_tokens=lowerCAmelCase_ )

														# To avoid mixing byte-level and unicode for byte-level BPT
														# we need to build string separately for added tokens and byte-level tokens
														# cf. https://github.com/huggingface/transformers/issues/1133
														_a     					=    []
														_a     					=    []
														for token in filtered_tokens:
																					if skip_special_tokens and token in self.all_special_ids:
																												continue
																					if token in self.added_tokens_encoder:
																												if current_sub_text:
																																			sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase_ ) )
																																			_a     					=    []
																												sub_texts.append(lowerCAmelCase_ )
																					else:
																												current_sub_text.append(lowerCAmelCase_ )
														if current_sub_text:
																					sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase_ ) )

														# Mimic the behavior of the Rust tokenizer:
														# No space before [MASK] and [SEP]
														if spaces_between_special_tokens:
																					_a     					=    re.sub(R''' (\[(MASK|SEP)\])'''							,      R'''\1'''							,      ''' '''.join(lowerCAmelCase_ ) )
														else:
																					_a     					=    ''''''.join(lowerCAmelCase_ )

														_a     					=    (
														    clean_up_tokenization_spaces
														    if clean_up_tokenization_spaces is not None
														    else self.clean_up_tokenization_spaces
														)
														if clean_up_tokenization_spaces:
																					_a     					=    self.clean_up_tokenization(lowerCAmelCase_ )
																					return clean_text
														else:
																					return text







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[str] = None )      ->							Tuple[str]:


														"""simple docstring"""


														if not os.path.isdir(lowerCAmelCase_ ):
																					logger.error(F'Vocabulary path ({save_directory}) should be a directory' )
																					return
														_a     					=    os.path.join(
														    lowerCAmelCase_							,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

														if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCAmelCase_ ) and os.path.isfile(self.vocab_file ):
																					copyfile(self.vocab_file							,      lowerCAmelCase_ )
														elif not os.path.isfile(self.vocab_file ):
																					with open(lowerCAmelCase_							,      '''wb''' ) as fi:
																												_a     					=    self.sp_model.serialized_model_proto()
																												fi.write(lowerCAmelCase_ )

														return (out_vocab_file,)







							def        __lowerCAmelCase      (  self	:  int							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														_a     					=    [self.sep_token_id]
														return cls + token_ids_a + sep + token_ids_a + sep







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None							,      lowerCAmelCase_	:  bool = False )      ->							List[int]:


														"""simple docstring"""


														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=lowerCAmelCase_							,      token_ids_a=lowerCAmelCase_							,      already_has_special_tokens=lowerCAmelCase_ )

														if token_ids_a is None:
																					return [1] + ([0] * len(lowerCAmelCase_ )) + [1]
														return [1] + ([0] * len(lowerCAmelCase_ )) + [1] + ([0] * len(lowerCAmelCase_ )) + [1]







							def        __lowerCAmelCase      (  self	:  Tuple							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep ) * [0]
														return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







'''simple docstring'''


import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule

class       A   (		_a   ):
							lowercase_          =    42
							lowercase_          =    jnp.floataa
							lowercase_          =    True







							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							Dict:


														"""simple docstring"""


														super().setup()
														_a     					=    nn.Dense(5							,      dtype=self.dtype )







							def __call__(  self	:  List[Any]							,      *lowerCAmelCase_	:  str							,      **lowerCAmelCase_	:  Dict )      ->							List[Any]:


														"""simple docstring"""


														_a     					=    super().__call__(*lowerCAmelCase_							,      **lowerCAmelCase_ )
														_a     					=    self.cls(outputs[2] )
														return outputs[:2] + (cls_out,)



class       A   (		_a   ):
							lowercase_          =    FlaxBigBirdForNaturalQuestionsModule



def 		snake_case_   (UpperCamelCase	:  Any			,						UpperCamelCase	:  str			,						UpperCamelCase	:  int			,						UpperCamelCase	:  Tuple			,						UpperCamelCase	:  Union[str, Any]			,						UpperCamelCase	:  int				):





							'''simple docstring'''


							def cross_entropy(UpperCamelCase	:  int			,						UpperCamelCase	:  Optional[Any]			,						UpperCamelCase	:  List[str]=None				):
														_a     					=    logits.shape[-1]
														_a     					=    (labels[..., None] == jnp.arange(UpperCamelCase				)[None]).astype('''f4'''				)
														_a     					=    jax.nn.log_softmax(UpperCamelCase			,						axis=-1				)
														_a     					=    -jnp.sum(labels * logits			,						axis=-1				)
														if reduction is not None:
																					_a     					=    reduction(UpperCamelCase				)
														return loss

							_a     					=    partial(UpperCamelCase			,						reduction=jnp.mean				)
							_a     					=    cross_entropy(UpperCamelCase			,						UpperCamelCase				)
							_a     					=    cross_entropy(UpperCamelCase			,						UpperCamelCase				)
							_a     					=    cross_entropy(UpperCamelCase			,						UpperCamelCase				)
							return (start_loss + end_loss + pooled_loss) / 3

@dataclass
class       A   :
							lowercase_          =    "google/bigbird-roberta-base"
							lowercase_          =    3000
							lowercase_          =    1_0500

							lowercase_          =    128
							lowercase_          =    3

							lowercase_          =    1
							lowercase_          =    5

							# tx_args
							lowercase_          =    3e-5
							lowercase_          =    0.0
							lowercase_          =    2_0000
							lowercase_          =    0.0095

							lowercase_          =    "bigbird-roberta-natural-questions"
							lowercase_          =    "training-expt"
							lowercase_          =    "data/nq-training.jsonl"
							lowercase_          =    "data/nq-validation.jsonl"







							def        __lowerCAmelCase      (  self	:  Union[str, Any] )      ->							Any:


														"""simple docstring"""


														os.makedirs(self.base_dir							,      exist_ok=lowerCAmelCase_ )
														_a     					=    os.path.join(self.base_dir							,      self.save_dir )
														_a     					=    self.batch_size_per_device * jax.device_count()



@dataclass
class       A   :
							lowercase_          =    42
							lowercase_          =    4096  # no dynamic padding on TPUs







							def __call__(  self	:  str							,      lowerCAmelCase_	:  Union[str, Any] )      ->							Union[str, Any]:


														"""simple docstring"""


														_a     					=    self.collate_fn(lowerCAmelCase_ )
														_a     					=    jax.tree_util.tree_map(lowerCAmelCase_							,      lowerCAmelCase_ )
														return batch







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  List[Any] )      ->							int:


														"""simple docstring"""


														_a					,							_a     					=    self.fetch_inputs(features['''input_ids'''] )
														_a     					=    {
														    '''input_ids''': jnp.array(lowerCAmelCase_							,      dtype=jnp.intaa ),
														    '''attention_mask''': jnp.array(lowerCAmelCase_							,      dtype=jnp.intaa ),
														    '''start_labels''': jnp.array(features['''start_token''']							,      dtype=jnp.intaa ),
														    '''end_labels''': jnp.array(features['''end_token''']							,      dtype=jnp.intaa ),
														    '''pooled_labels''': jnp.array(features['''category''']							,      dtype=jnp.intaa ),
														}
														return batch







							def        __lowerCAmelCase      (  self	:  Dict							,      lowerCAmelCase_	:  list )      ->							List[Any]:


														"""simple docstring"""


														_a     					=    [self._fetch_inputs(lowerCAmelCase_ ) for ids in input_ids]
														return zip(*lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Dict							,      lowerCAmelCase_	:  list )      ->							str:


														"""simple docstring"""


														_a     					=    [1 for _ in range(len(lowerCAmelCase_ ) )]
														while len(lowerCAmelCase_ ) < self.max_length:
																					input_ids.append(self.pad_id )
																					attention_mask.append(0 )
														return input_ids, attention_mask



def 		snake_case_   (UpperCamelCase	:  List[Any]			,						UpperCamelCase	:  List[Any]			,						UpperCamelCase	:  Dict=None				):





							'''simple docstring'''


							if seed is not None:
														_a     					=    dataset.shuffle(seed=UpperCamelCase				)
							for i in range(len(UpperCamelCase				) // batch_size				):
														_a     					=    dataset[i * batch_size : (i + 1) * batch_size]
														yield dict(UpperCamelCase				)
@partial(jax.pmap			,						axis_name='''batch'''				)
def 		snake_case_   (UpperCamelCase	:  Optional[int]			,						UpperCamelCase	:  Tuple			,						**UpperCamelCase	:  str				):





							'''simple docstring'''


							def loss_fn(UpperCamelCase	:  List[str]				):
														_a     					=    model_inputs.pop('''start_labels'''				)
														_a     					=    model_inputs.pop('''end_labels'''				)
														_a     					=    model_inputs.pop('''pooled_labels'''				)

														_a     					=    state.apply_fn(**UpperCamelCase			,						params=UpperCamelCase			,						dropout_rng=UpperCamelCase			,						train=UpperCamelCase				)
														_a					,							_a					,							_a     					=    outputs

														return state.loss_fn(
														    UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase			,						)

							_a					,							_a     					=    jax.random.split(UpperCamelCase				)
							_a     					=    jax.value_and_grad(UpperCamelCase				)
							_a					,							_a     					=    grad_fn(state.params				)
							_a     					=    jax.lax.pmean({'''loss''': loss}			,						axis_name='''batch'''				)
							_a     					=    jax.lax.pmean(UpperCamelCase			,						'''batch'''				)

							_a     					=    state.apply_gradients(grads=UpperCamelCase				)
							return state, metrics, new_drp_rng
@partial(jax.pmap			,						axis_name='''batch'''				)
def 		snake_case_   (UpperCamelCase	:  Any			,						**UpperCamelCase	:  Optional[int]				):





							'''simple docstring'''


							_a     					=    model_inputs.pop('''start_labels'''				)
							_a     					=    model_inputs.pop('''end_labels'''				)
							_a     					=    model_inputs.pop('''pooled_labels'''				)

							_a     					=    state.apply_fn(**UpperCamelCase			,						params=state.params			,						train=UpperCamelCase				)
							_a					,							_a					,							_a     					=    outputs

							_a     					=    state.loss_fn(UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase				)
							_a     					=    jax.lax.pmean({'''loss''': loss}			,						axis_name='''batch'''				)
							return metrics

class       A   (		train_state.TrainState   ):
							lowercase_          =    struct.field(pytree_node=_a   )



@dataclass
class       A   :
							lowercase_          =    42
							lowercase_          =    42
							lowercase_          =    42
							lowercase_          =    42
							lowercase_          =    42
							lowercase_          =    42
							lowercase_          =    None







							def        __lowerCAmelCase      (  self	:  Tuple							,      lowerCAmelCase_	:  Optional[Any]							,      lowerCAmelCase_	:  Optional[Any]							,      lowerCAmelCase_	:  Optional[Any]							,      lowerCAmelCase_	:  Optional[int]=None )      ->							List[str]:


														"""simple docstring"""


														_a     					=    model.params
														_a     					=    TrainState.create(
														    apply_fn=model.__call__							,      params=lowerCAmelCase_							,      tx=lowerCAmelCase_							,      loss_fn=lowerCAmelCase_							,      )
														if ckpt_dir is not None:
																					_a					,							_a					,							_a					,							_a					,							_a     					=    restore_checkpoint(lowerCAmelCase_							,      lowerCAmelCase_ )
																					_a     					=    {
																					    '''lr''': args.lr,
																					    '''init_lr''': args.init_lr,
																					    '''warmup_steps''': args.warmup_steps,
																					    '''num_train_steps''': num_train_steps,
																					    '''weight_decay''': args.weight_decay,
																					}
																					_a					,							_a     					=    build_tx(**lowerCAmelCase_ )
																					_a     					=    train_state.TrainState(
																					    step=lowerCAmelCase_							,      apply_fn=model.__call__							,      params=lowerCAmelCase_							,      tx=lowerCAmelCase_							,      opt_state=lowerCAmelCase_							,      )
																					_a     					=    args
																					_a     					=    data_collator
																					_a     					=    lr
																					_a     					=    params
														_a     					=    jax_utils.replicate(lowerCAmelCase_ )
														return state







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  Any							,      lowerCAmelCase_	:  Union[str, Any]							,      lowerCAmelCase_	:  Optional[Any] )      ->							int:


														"""simple docstring"""


														_a     					=    self.args
														_a     					=    len(lowerCAmelCase_ ) // args.batch_size

														_a     					=    jax.random.PRNGKey(0 )
														_a     					=    jax.random.split(lowerCAmelCase_							,      jax.device_count() )
														for epoch in range(args.max_epochs ):
																					_a     					=    jnp.array(0							,      dtype=jnp.floataa )
																					_a     					=    get_batched_dataset(lowerCAmelCase_							,      args.batch_size							,      seed=lowerCAmelCase_ )
																					_a     					=    0
																					for batch in tqdm(lowerCAmelCase_							,      total=lowerCAmelCase_							,      desc=F'Running EPOCH-{epoch}' ):
																												_a     					=    self.data_collator(lowerCAmelCase_ )
																												_a					,							_a					,							_a     					=    self.train_step_fn(lowerCAmelCase_							,      lowerCAmelCase_							,      **lowerCAmelCase_ )
																												running_loss += jax_utils.unreplicate(metrics['''loss'''] )
																												i += 1
																												if i % args.logging_steps == 0:
																																			_a     					=    jax_utils.unreplicate(state.step )
																																			_a     					=    running_loss.item() / i
																																			_a     					=    self.scheduler_fn(state_step - 1 )

																																			_a     					=    self.evaluate(lowerCAmelCase_							,      lowerCAmelCase_ )
																																			_a     					=    {
																																			    '''step''': state_step.item(),
																																			    '''eval_loss''': eval_loss.item(),
																																			    '''tr_loss''': tr_loss,
																																			    '''lr''': lr.item(),
																																			}
																																			tqdm.write(str(lowerCAmelCase_ ) )
																																			self.logger.log(lowerCAmelCase_							,      commit=lowerCAmelCase_ )

																												if i % args.save_steps == 0:
																																			self.save_checkpoint(args.save_dir + F'-e{epoch}-s{i}'							,      state=lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  List[str]							,      lowerCAmelCase_	:  str )      ->							List[str]:


														"""simple docstring"""


														_a     					=    get_batched_dataset(lowerCAmelCase_							,      self.args.batch_size )
														_a     					=    len(lowerCAmelCase_ ) // self.args.batch_size
														_a     					=    jnp.array(0							,      dtype=jnp.floataa )
														_a     					=    0
														for batch in tqdm(lowerCAmelCase_							,      total=lowerCAmelCase_							,      desc='''Evaluating ... ''' ):
																					_a     					=    self.data_collator(lowerCAmelCase_ )
																					_a     					=    self.val_step_fn(lowerCAmelCase_							,      **lowerCAmelCase_ )
																					running_loss += jax_utils.unreplicate(metrics['''loss'''] )
																					i += 1
														return running_loss / i







							def        __lowerCAmelCase      (  self	:  int							,      lowerCAmelCase_	:  List[Any]							,      lowerCAmelCase_	:  str )      ->							int:


														"""simple docstring"""


														_a     					=    jax_utils.unreplicate(lowerCAmelCase_ )
														print(F'SAVING CHECKPOINT IN {save_dir}'							,      end=''' ... ''' )
														self.model_save_fn(lowerCAmelCase_							,      params=state.params )
														with open(os.path.join(lowerCAmelCase_							,      '''opt_state.msgpack''' )							,      '''wb''' ) as f:
																					f.write(to_bytes(state.opt_state ) )
														joblib.dump(self.args							,      os.path.join(lowerCAmelCase_							,      '''args.joblib''' ) )
														joblib.dump(self.data_collator							,      os.path.join(lowerCAmelCase_							,      '''data_collator.joblib''' ) )
														with open(os.path.join(lowerCAmelCase_							,      '''training_state.json''' )							,      '''w''' ) as f:
																					json.dump({'''step''': state.step.item()}							,      lowerCAmelCase_ )
														print('''DONE''' )



def 		snake_case_   (UpperCamelCase	:  int			,						UpperCamelCase	:  Dict				):





							'''simple docstring'''


							print(f'RESTORING CHECKPOINT FROM {save_dir}'			,						end=''' ... '''				)
							with open(os.path.join(UpperCamelCase			,						'''flax_model.msgpack'''				)			,						'''rb'''				) as f:
														_a     					=    from_bytes(state.params			,						f.read()				)

							with open(os.path.join(UpperCamelCase			,						'''opt_state.msgpack'''				)			,						'''rb'''				) as f:
														_a     					=    from_bytes(state.opt_state			,						f.read()				)

							_a     					=    joblib.load(os.path.join(UpperCamelCase			,						'''args.joblib'''				)				)
							_a     					=    joblib.load(os.path.join(UpperCamelCase			,						'''data_collator.joblib'''				)				)

							with open(os.path.join(UpperCamelCase			,						'''training_state.json'''				)			,						'''r'''				) as f:
														_a     					=    json.load(UpperCamelCase				)
							_a     					=    training_state['''step''']

							print('''DONE'''				)
							return params, opt_state, step, args, data_collator
def 		snake_case_   (UpperCamelCase	:  Dict			,						UpperCamelCase	:  Optional[int]			,						UpperCamelCase	:  Dict			,						UpperCamelCase	:  List[Any]				):





							'''simple docstring'''


							_a     					=    num_train_steps - warmup_steps
							_a     					=    optax.linear_schedule(init_value=UpperCamelCase			,						end_value=UpperCamelCase			,						transition_steps=UpperCamelCase				)
							_a     					=    optax.linear_schedule(init_value=UpperCamelCase			,						end_value=1e-7			,						transition_steps=UpperCamelCase				)
							_a     					=    optax.join_schedules(schedules=[warmup_fn, decay_fn]			,						boundaries=[warmup_steps]				)
							return lr

def 		snake_case_   (UpperCamelCase	:  Tuple			,						UpperCamelCase	:  Optional[Any]			,						UpperCamelCase	:  str			,						UpperCamelCase	:  Union[str, Any]			,						UpperCamelCase	:  int				):





							'''simple docstring'''


							def weight_decay_mask(UpperCamelCase	:  Dict				):
														_a     					=    traverse_util.flatten_dict(UpperCamelCase				)
														_a     					=    {k: (v[-1] != '''bias''' and v[-2:] != ('''LayerNorm''', '''scale''')) for k, v in params.items()}
														return traverse_util.unflatten_dict(UpperCamelCase				)

							_a     					=    scheduler_fn(UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase			,						UpperCamelCase				)

							_a     					=    optax.adamw(learning_rate=UpperCamelCase			,						weight_decay=UpperCamelCase			,						mask=UpperCamelCase				)
							return tx, lr

