





'''simple docstring'''






from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class 			UpperCAmelCase					(      snake_case_							,		snake_case_							,		snake_case_       ):
      _lowercase:      List[Any]											=       [r'''h\.\d+\.attn\.bias''', r'''h\.\d+\.attn\.masked_bias''']




      @register_to_config
      def __init__(      self      :			Optional[int]      ,      __snake_case      :			int      ,      __snake_case      :			int      ,      __snake_case      :			Optional[int] = None      ,      __snake_case      :			int = 5_02_57      ,      __snake_case      :			int = 10_24      ,      __snake_case      :			int = 7_68      ,      __snake_case      :			int = 12      ,      __snake_case      :			int = 12      ,      __snake_case      :			Optional[int] = None      ,      __snake_case      :			str = "gelu_new"      ,      __snake_case      :			float = 0.1      ,      __snake_case      :			float = 0.1      ,      __snake_case      :			float = 0.1      ,      __snake_case      :			float = 1E-5      ,      __snake_case      :			float = 0.02      ,      __snake_case      :			bool = True      ,      __snake_case      :			bool = True      ,      __snake_case      :			bool = False      ,      __snake_case      :			bool = False      ,      )    ->				int:
            super().__init__()

            _lowerCAmelCase												=							prefix_length

            if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
                  raise ValueError(
                      f"`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and"
                      f" `n_embd`: {n_embd} are not equal."  )

            _lowerCAmelCase												=							prefix_inner_dim
            _lowerCAmelCase												=							prefix_hidden_dim

            _lowerCAmelCase												=							(
                nn.Linear(self.prefix_inner_dim      ,      self.prefix_hidden_dim  )
                if self.prefix_hidden_dim is not None
                else nn.Identity()
            )
            _lowerCAmelCase												=							(
                nn.Linear(self.prefix_hidden_dim      ,      __snake_case  ) if self.prefix_hidden_dim is not None else nn.Identity()
            )

            _lowerCAmelCase												=							GPTaConfig(
                vocab_size=__snake_case      ,      n_positions=__snake_case      ,      n_embd=__snake_case      ,      n_layer=__snake_case      ,      n_head=__snake_case      ,      n_inner=__snake_case      ,      activation_function=__snake_case      ,      resid_pdrop=__snake_case      ,      embd_pdrop=__snake_case      ,      attn_pdrop=__snake_case      ,      layer_norm_epsilon=__snake_case      ,      initializer_range=__snake_case      ,      scale_attn_weights=__snake_case      ,      use_cache=__snake_case      ,      scale_attn_by_inverse_layer_idx=__snake_case      ,      reorder_and_upcast_attn=__snake_case      ,      )
            _lowerCAmelCase												=							GPTaLMHeadModel(__snake_case  )




      def     lowercase__   (      self      :			Optional[int]      ,      __snake_case      :			torch.Tensor      ,      __snake_case      :			torch.Tensor      ,      __snake_case      :			Optional[torch.Tensor] = None      ,      __snake_case      :			Optional[torch.Tensor] = None      ,      )    ->				Dict:
            _lowerCAmelCase												=							self.transformer.transformer.wte(__snake_case  )
            _lowerCAmelCase												=							self.encode_prefix(__snake_case  )
            _lowerCAmelCase												=							self.decode_prefix(__snake_case  )
            _lowerCAmelCase												=							torch.cat((prefix_embeds, embedding_text)      ,      dim=1  )

            if labels is not None:
                  _lowerCAmelCase												=							self.get_dummy_token(input_ids.shape[0]      ,      input_ids.device  )
                  _lowerCAmelCase												=							torch.cat((dummy_token, input_ids)      ,      dim=1  )
            _lowerCAmelCase												=							self.transformer(inputs_embeds=__snake_case      ,      labels=__snake_case      ,      attention_mask=__snake_case  )
            if self.prefix_hidden_dim is not None:
                  return out, hidden
            else:
                  return out




      def     lowercase__   (      self      :			Dict      ,      __snake_case      :			int      ,      __snake_case      :			torch.device  )    ->				torch.Tensor:
            return torch.zeros(__snake_case      ,      self.prefix_length      ,      dtype=torch.intaa      ,      device=__snake_case  )




      def     lowercase__   (      self      :			Optional[int]      ,      __snake_case      :			List[Any]  )    ->				Union[str, Any]:
            return self.encode_prefix(__snake_case  )




      @torch.no_grad()
      def     lowercase__   (      self      :			List[str]      ,      __snake_case      :			int      ,      __snake_case      :			List[Any]      ,      __snake_case      :			Dict  )    ->				List[str]:
            _lowerCAmelCase												=							torch.split(__snake_case      ,      1      ,      dim=0  )
            _lowerCAmelCase												=							[]
            _lowerCAmelCase												=							[]
            for feature in features:
                  _lowerCAmelCase												=							self.decode_prefix(feature.to(__snake_case  )  )  # back to the clip feature
                  # Only support beam search for now
                  _lowerCAmelCase					,							_lowerCAmelCase												=							self.generate_beam(
                      input_embeds=__snake_case      ,      device=__snake_case      ,      eos_token_id=__snake_case  )
                  generated_tokens.append(output_tokens[0]  )
                  generated_seq_lengths.append(seq_lengths[0]  )
            _lowerCAmelCase												=							torch.stack(__snake_case  )
            _lowerCAmelCase												=							torch.stack(__snake_case  )
            return generated_tokens, generated_seq_lengths




      @torch.no_grad()
      def     lowercase__   (      self      :			Optional[int]      ,      __snake_case      :			List[Any]=None      ,      __snake_case      :			str=None      ,      __snake_case      :			List[str]=None      ,      __snake_case      :			int = 5      ,      __snake_case      :			int = 67      ,      __snake_case      :			float = 1.0      ,      __snake_case      :			Optional[int] = None      ,      )    ->				Union[str, Any]:
            _lowerCAmelCase												=							eos_token_id
            _lowerCAmelCase												=							None
            _lowerCAmelCase												=							None
            _lowerCAmelCase												=							torch.ones(__snake_case      ,      device=__snake_case      ,      dtype=torch.int  )
            _lowerCAmelCase												=							torch.zeros(__snake_case      ,      device=__snake_case      ,      dtype=torch.bool  )

            if input_embeds is not None:
                  _lowerCAmelCase												=							input_embeds
            else:
                  _lowerCAmelCase												=							self.transformer.transformer.wte(__snake_case  )

            for i in range(__snake_case  ):
                  _lowerCAmelCase												=							self.transformer(inputs_embeds=__snake_case  )
                  _lowerCAmelCase												=							outputs.logits
                  _lowerCAmelCase												=							logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
                  _lowerCAmelCase												=							logits.softmax(-1  ).log()

                  if scores is None:
                        _lowerCAmelCase					,							_lowerCAmelCase												=							logits.topk(__snake_case      ,      -1  )
                        _lowerCAmelCase												=							generated.expand(__snake_case      ,      *generated.shape[1:]  )
                        _lowerCAmelCase					,							_lowerCAmelCase												=							next_tokens.permute(1      ,      0  ), scores.squeeze(0  )
                        if tokens is None:
                              _lowerCAmelCase												=							next_tokens
                        else:
                              _lowerCAmelCase												=							tokens.expand(__snake_case      ,      *tokens.shape[1:]  )
                              _lowerCAmelCase												=							torch.cat((tokens, next_tokens)      ,      dim=1  )
                  else:
                        _lowerCAmelCase												=							-float(np.inf  )
                        _lowerCAmelCase												=							0
                        _lowerCAmelCase												=							scores[:, None] + logits
                        seq_lengths[~is_stopped] += 1
                        _lowerCAmelCase												=							scores_sum / seq_lengths[:, None]
                        _lowerCAmelCase					,							_lowerCAmelCase												=							scores_sum_average.view(-1  ).topk(__snake_case      ,      -1  )
                        _lowerCAmelCase												=							next_tokens // scores_sum.shape[1]
                        _lowerCAmelCase												=							seq_lengths[next_tokens_source]
                        _lowerCAmelCase												=							next_tokens % scores_sum.shape[1]
                        _lowerCAmelCase												=							next_tokens.unsqueeze(1  )
                        _lowerCAmelCase												=							tokens[next_tokens_source]
                        _lowerCAmelCase												=							torch.cat((tokens, next_tokens)      ,      dim=1  )
                        _lowerCAmelCase												=							generated[next_tokens_source]
                        _lowerCAmelCase												=							scores_sum_average * seq_lengths
                        _lowerCAmelCase												=							is_stopped[next_tokens_source]

                  _lowerCAmelCase												=							self.transformer.transformer.wte(next_tokens.squeeze()  ).view(generated.shape[0]      ,      1      ,      -1  )
                  _lowerCAmelCase												=							torch.cat((generated, next_token_embed)      ,      dim=1  )
                  _lowerCAmelCase												=							is_stopped + next_tokens.eq(__snake_case  ).squeeze()
                  if is_stopped.all():
                        break

            _lowerCAmelCase												=							scores / seq_lengths
            _lowerCAmelCase												=							scores.argsort(descending=__snake_case  )
            # tokens tensors are already padded to max_seq_length
            _lowerCAmelCase												=							[tokens[i] for i in order]
            _lowerCAmelCase												=							torch.stack(__snake_case      ,      dim=0  )
            _lowerCAmelCase												=							torch.tensor([seq_lengths[i] for i in order]      ,      dtype=seq_lengths.dtype  )
            return output_texts, seq_lengths













'''simple docstring'''






import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


A__    :       str              =5_00_00
A__    :       Optional[int]              =50_00

A__       ,						A__    :       Optional[int]              =os.path.split(__file__)
A__    :       Tuple              =os.path.join(RESULTS_BASEPATH, '''results''', RESULTS_FILENAME.replace('''.py''', '''.json'''))
@get_duration
def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase  ):





      """simple docstring"""


      for i in range(lowerCAmelCase  ):
            _lowerCAmelCase												=							dataset[i]
@get_duration
def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase  ):





      """simple docstring"""


      for i in range(0							,     len(lowerCAmelCase  )							,     lowerCAmelCase  ):
            _lowerCAmelCase												=							dataset[i : i + batch_size]
@get_duration
def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase  ):





      """simple docstring"""


      with dataset.formatted_as(type=lowerCAmelCase  ):
            for i in range(lowerCAmelCase  ):
                  _lowerCAmelCase												=							dataset[i]
@get_duration
def        UpperCamelCase__				(		lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase  ):





      """simple docstring"""


      with dataset.formatted_as(type=lowerCAmelCase  ):
            for i in range(0							,     lowerCAmelCase							,     lowerCAmelCase  ):
                  _lowerCAmelCase												=							dataset[i : i + batch_size]
def        UpperCamelCase__				(		):





      """simple docstring"""


      _lowerCAmelCase												=							{"""num examples""": SPEED_TEST_N_EXAMPLES}
      _lowerCAmelCase												=							[
          (read, {"""length""": SMALL_TEST}),
          (read, {"""length""": SPEED_TEST_N_EXAMPLES}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 10}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 1_00}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 10_00}),
          (read_formatted, {"""type""": """numpy""", """length""": SMALL_TEST}),
          (read_formatted, {"""type""": """pandas""", """length""": SMALL_TEST}),
          (read_formatted, {"""type""": """torch""", """length""": SMALL_TEST}),
          (read_formatted, {"""type""": """tensorflow""", """length""": SMALL_TEST}),
          (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 10}),
          (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 10_00}),
      ]

      _lowerCAmelCase												=							[
          (read, {"""length""": SMALL_TEST}),
          (read, {"""length""": SPEED_TEST_N_EXAMPLES}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 10}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 1_00}),
          (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 10_00}),
          (read_formatted, {"""type""": """numpy""", """length""": SMALL_TEST}),
          (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 10}),
          (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 10_00}),
      ]
      with tempfile.TemporaryDirectory() as tmp_dir:
            print("""generating dataset"""  )
            _lowerCAmelCase												=							datasets.Features(
                {"""list""": datasets.Sequence(datasets.Value("""float32"""  )  ), """numbers""": datasets.Value("""float32"""  )}  )
            _lowerCAmelCase												=							generate_example_dataset(
                os.path.join(lowerCAmelCase							,     """dataset.arrow"""  )							,     lowerCAmelCase							,     num_examples=lowerCAmelCase							,     seq_shapes={"""list""": (1_00,)}							,     )
            print("""first set of iterations"""  )
            for func, kwargs in functions:
                  print(func.__name__							,     str(lowerCAmelCase  )  )
                  _lowerCAmelCase												=							func(lowerCAmelCase							,     **lowerCAmelCase  )

            print("""shuffling dataset"""  )
            _lowerCAmelCase												=							dataset.shuffle()
            print("""Second set of iterations (after shuffling"""  )
            for func, kwargs in functions_shuffled:
                  print("""shuffled """							,     func.__name__							,     str(lowerCAmelCase  )  )
                  _lowerCAmelCase												=							func(
                      lowerCAmelCase							,     **lowerCAmelCase  )

      with open(lowerCAmelCase							,     """wb"""  ) as f:
            f.write(json.dumps(lowerCAmelCase  ).encode("""utf-8"""  )  )


if __name__ == "__main__":  # useful to run the profiler
      benchmark_iterating()






