


"""simple docstring"""
import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE							:      Optional[int]						=      logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							:      List[Any]						=      {
    'vocab_file': 'vocab.txt',
    'merges_file': 'bpe.codes',
}

__SCREAMING_SNAKE_CASE							:      str						=      {
    'vocab_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt',
    },
    'merges_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes',
    },
}

__SCREAMING_SNAKE_CASE							:      Dict						=      {
    'vinai/phobert-base': 256,
    'vinai/phobert-large': 256,
}







def   _a			( _SCREAMING_SNAKE_CASE			)      ->			Tuple:
							snake_case_          =						set()
							snake_case_          =						word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)			)
														snake_case_          =						char

							snake_case_          =						set(_SCREAMING_SNAKE_CASE			)
							return pairs





class        __A       (snake_case__):





							'''simple docstring'''

							__lowercase:						int							 =      VOCAB_FILES_NAMES
							__lowercase:						List[Any]							 =      PRETRAINED_VOCAB_FILES_MAP
							__lowercase:						str							 =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



							def __init__(			self							: Tuple						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: Dict						,    UpperCAmelCase_							: Optional[Any]="<s>"						,    UpperCAmelCase_							: str="</s>"						,    UpperCAmelCase_							: int="</s>"						,    UpperCAmelCase_							: int="<s>"						,    UpperCAmelCase_							: Any="<unk>"						,    UpperCAmelCase_							: Any="<pad>"						,    UpperCAmelCase_							: Dict="<mask>"						,    **UpperCAmelCase_							: Optional[Any]						,    )   ->Optional[Any]:



														"""simple docstring"""



														super().__init__(
														    bos_token=UpperCAmelCase_						,    eos_token=UpperCAmelCase_						,    unk_token=UpperCAmelCase_						,    sep_token=UpperCAmelCase_						,    cls_token=UpperCAmelCase_						,    pad_token=UpperCAmelCase_						,    mask_token=UpperCAmelCase_						,    **UpperCAmelCase_						,    )

														snake_case_          =						vocab_file
														snake_case_          =						merges_file

														snake_case_          =						{}
														snake_case_          =						0
														snake_case_          =						1
														snake_case_          =						2
														snake_case_          =						3

														self.add_from_file(UpperCAmelCase_    )

														snake_case_          =						{v: k for k, v in self.encoder.items()}

														with open(UpperCAmelCase_						,    encoding="""utf-8"""    ) as merges_handle:
																					snake_case_          =						merges_handle.read().split("""\n"""    )[:-1]
														snake_case_          =						[tuple(merge.split()[:-1]    ) for merge in merges]
														snake_case_          =						dict(zip(UpperCAmelCase_						,    range(len(UpperCAmelCase_    )    )    )    )
														snake_case_          =						{}



							def 						lowerCAmelCase    (			self							: int						,    UpperCAmelCase_							: List[int]						,    UpperCAmelCase_							: Optional[List[int]] = None    )   ->List[int]:



														"""simple docstring"""



														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														snake_case_          =						[self.cls_token_id]
														snake_case_          =						[self.sep_token_id]
														return cls + token_ids_a + sep + sep + token_ids_a + sep



							def 						lowerCAmelCase    (			self							: str						,    UpperCAmelCase_							: List[int]						,    UpperCAmelCase_							: Optional[List[int]] = None						,    UpperCAmelCase_							: bool = False    )   ->List[int]:



														"""simple docstring"""



														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=UpperCAmelCase_						,    token_ids_a=UpperCAmelCase_						,    already_has_special_tokens=UpperCAmelCase_    )

														if token_ids_a is None:
																					return [1] + ([0] * len(UpperCAmelCase_    )) + [1]
														return [1] + ([0] * len(UpperCAmelCase_    )) + [1, 1] + ([0] * len(UpperCAmelCase_    )) + [1]



							def 						lowerCAmelCase    (			self							: List[str]						,    UpperCAmelCase_							: List[int]						,    UpperCAmelCase_							: Optional[List[int]] = None    )   ->List[int]:



														"""simple docstring"""



														snake_case_          =						[self.sep_token_id]
														snake_case_          =						[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep    ) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]



							@property
							def 						lowerCAmelCase    (			self							: Any    )   ->Tuple:



														"""simple docstring"""



														return len(self.encoder    )



							def 						lowerCAmelCase    (			self							: int    )   ->List[Any]:



														"""simple docstring"""



														return dict(self.encoder						,    **self.added_tokens_encoder    )



							def 						lowerCAmelCase    (			self							: Tuple						,    UpperCAmelCase_							: List[Any]    )   ->Any:



														"""simple docstring"""



														if token in self.cache:
																					return self.cache[token]
														snake_case_          =						tuple(UpperCAmelCase_    )
														snake_case_          =						tuple(list(word[:-1]    ) + [word[-1] + """</w>"""]    )
														snake_case_          =						get_pairs(UpperCAmelCase_    )

														if not pairs:
																					return token

														while True:
																					snake_case_          =						min(UpperCAmelCase_						,    key=lambda UpperCAmelCase_    : self.bpe_ranks.get(UpperCAmelCase_						,    float("""inf"""    )    )    )
																					if bigram not in self.bpe_ranks:
																												break
																					snake_case_    ,  snake_case_          =						bigram
																					snake_case_          =						[]
																					snake_case_          =						0
																					while i < len(UpperCAmelCase_    ):
																												try:
																																			snake_case_          =						word.index(UpperCAmelCase_						,    UpperCAmelCase_    )
																												except ValueError:
																																			new_word.extend(word[i:]    )
																																			break
																												else:
																																			new_word.extend(word[i:j]    )
																																			snake_case_          =						j

																												if word[i] == first and i < len(UpperCAmelCase_    ) - 1 and word[i + 1] == second:
																																			new_word.append(first + second    )
																																			i += 2
																												else:
																																			new_word.append(word[i]    )
																																			i += 1
																					snake_case_          =						tuple(UpperCAmelCase_    )
																					snake_case_          =						new_word
																					if len(UpperCAmelCase_    ) == 1:
																												break
																					else:
																												snake_case_          =						get_pairs(UpperCAmelCase_    )
														snake_case_          =						"""@@ """.join(UpperCAmelCase_    )
														snake_case_          =						word[:-4]
														snake_case_          =						word
														return word



							def 						lowerCAmelCase    (			self							: str						,    UpperCAmelCase_							: List[str]    )   ->Tuple:



														"""simple docstring"""



														snake_case_          =						[]

														snake_case_          =						re.findall(R"""\S+\n?"""						,    UpperCAmelCase_    )

														for token in words:
																					split_tokens.extend(list(self.bpe(UpperCAmelCase_    ).split(""" """    )    )    )
														return split_tokens



							def 						lowerCAmelCase    (			self							: Tuple						,    UpperCAmelCase_							: str    )   ->Any:



														"""simple docstring"""



														return self.encoder.get(UpperCAmelCase_						,    self.encoder.get(self.unk_token    )    )



							def 						lowerCAmelCase    (			self							: List[str]						,    UpperCAmelCase_							: Any    )   ->Tuple:



														"""simple docstring"""



														return self.decoder.get(UpperCAmelCase_						,    self.unk_token    )



							def 						lowerCAmelCase    (			self							: Any						,    UpperCAmelCase_							: Optional[Any]    )   ->Dict:



														"""simple docstring"""



														snake_case_          =						""" """.join(UpperCAmelCase_    ).replace("""@@ """						,    """"""    ).strip()
														return out_string



							def 						lowerCAmelCase    (			self							: Dict						,    UpperCAmelCase_							: str						,    UpperCAmelCase_							: Optional[str] = None    )   ->Tuple[str]:



														"""simple docstring"""



														if not os.path.isdir(UpperCAmelCase_    ):
																					logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
																					return
														snake_case_          =						os.path.join(
														    UpperCAmelCase_						,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
														snake_case_          =						os.path.join(
														    UpperCAmelCase_						,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]    )

														if os.path.abspath(self.vocab_file    ) != os.path.abspath(UpperCAmelCase_    ):
																					copyfile(self.vocab_file						,    UpperCAmelCase_    )

														if os.path.abspath(self.merges_file    ) != os.path.abspath(UpperCAmelCase_    ):
																					copyfile(self.merges_file						,    UpperCAmelCase_    )

														return out_vocab_file, out_merge_file



							def 						lowerCAmelCase    (			self							: Tuple						,    UpperCAmelCase_							: Dict    )   ->Dict:



														"""simple docstring"""



														if isinstance(UpperCAmelCase_						,    UpperCAmelCase_    ):
																					try:
																												with open(UpperCAmelCase_						,    """r"""						,    encoding="""utf-8"""    ) as fd:
																																			self.add_from_file(UpperCAmelCase_    )
																					except FileNotFoundError as fnfe:
																												raise fnfe
																					except UnicodeError:
																												raise Exception(F"""Incorrect encoding detected in {f}, please rebuild the dataset"""    )
																					return

														snake_case_          =						f.readlines()
														for lineTmp in lines:
																					snake_case_          =						lineTmp.strip()
																					snake_case_          =						line.rfind(""" """    )
																					if idx == -1:
																												raise ValueError("""Incorrect dictionary format, expected '<token> <cnt>'"""    )
																					snake_case_          =						line[:idx]
																					snake_case_          =						len(self.encoder    )







"""simple docstring"""
import faiss  # noqa: F401 # Here to have a nice missing dependency error message early on
import numpy  # noqa: F401 # Here to have a nice missing dependency error message early on
import requests  # noqa: F401 # Here to have a nice missing dependency error message early on
import sklearn  # noqa: F401 # Here to have a nice missing dependency error message early on
import tqdm  # noqa: F401 # Here to have a nice missing dependency error message early on
from mauve import compute_mauve  # From: mauve-text

import datasets


__SCREAMING_SNAKE_CASE							:      Dict						=      '\\n@inproceedings{pillutla-etal:mauve:neurips2021,\n  title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers},\n  author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid},\n  booktitle = {NeurIPS},\n  year      = {2021}\n}\n\n'

__SCREAMING_SNAKE_CASE							:      Optional[int]						=      '\\nMAUVE is a library built on PyTorch and HuggingFace Transformers to measure the gap between neural text and human text with the eponymous MAUVE measure.\n\nMAUVE summarizes both Type I and Type II errors measured softly using Kullback–Leibler (KL) divergences.\n\nFor details, see the MAUVE paper: https://arxiv.org/abs/2102.01454 (Neurips, 2021).\n\nThis metrics is a wrapper around the official implementation of MAUVE:\nhttps://github.com/krishnap25/mauve\n'

__SCREAMING_SNAKE_CASE							:      Tuple						=      '\nCalculates MAUVE scores between two lists of generated text and reference text.\nArgs:\n    predictions: list of generated text to score. Each predictions\n        should be a string with tokens separated by spaces.\n    references: list of reference for each prediction. Each\n        reference should be a string with tokens separated by spaces.\nOptional Args:\n    num_buckets: the size of the histogram to quantize P and Q. Options: \'auto\' (default) or an integer\n    pca_max_data: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. Default -1\n    kmeans_explained_var: amount of variance of the data to keep in dimensionality reduction by PCA. Default 0.9\n    kmeans_num_redo: number of times to redo k-means clustering (the best objective is kept). Default 5\n    kmeans_max_iter: maximum number of k-means iterations. Default 500\n    featurize_model_name: name of the model from which features are obtained. Default \'gpt2-large\' Use one of [\'gpt2\', \'gpt2-medium\', \'gpt2-large\', \'gpt2-xl\'].\n    device_id: Device for featurization. Supply a GPU id (e.g. 0 or 3) to use GPU. If no GPU with this id is found, use CPU\n    max_text_length: maximum number of tokens to consider. Default 1024\n    divergence_curve_discretization_size: Number of points to consider on the divergence curve. Default 25\n    mauve_scaling_factor: "c" from the paper. Default 5.\n    verbose: If True (default), print running time updates\n    seed: random seed to initialize k-means cluster assignments.\nReturns:\n    mauve: MAUVE score, a number between 0 and 1. Larger values indicate that P and Q are closer,\n    frontier_integral: Frontier Integral, a number between 0 and 1. Smaller values indicate that P and Q are closer,\n    divergence_curve: a numpy.ndarray of shape (m, 2); plot it with matplotlib to view the divergence curve,\n    p_hist: a discrete distribution, which is a quantized version of the text distribution p_text,\n    q_hist: same as above, but with q_text.\nExamples:\n\n    >>> # faiss segfaults in doctest for some reason, so the .compute call is not tested with doctest\n    >>> import datasets\n    >>> mauve = datasets.load_metric(\'mauve\')\n    >>> predictions = ["hello there", "general kenobi"]\n    >>> references = ["hello there", "general kenobi"]\n    >>> out = mauve.compute(predictions=predictions, references=references) # doctest: +SKIP\n    >>> print(out.mauve) # doctest: +SKIP\n    1.0\n'





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,						_KWARGS_DESCRIPTION)
class        __A       (datasets.Metric):





							'''simple docstring'''

							def 						lowerCAmelCase    (			self							: Tuple    )   ->Union[str, Any]:



														"""simple docstring"""



														return datasets.MetricInfo(
														    description=_DESCRIPTION						,    citation=_CITATION						,    homepage="""https://github.com/krishnap25/mauve"""						,    inputs_description=_KWARGS_DESCRIPTION						,    features=datasets.Features(
														        {
														            """predictions""": datasets.Value("""string"""						,    id="""sequence"""    ),
														            """references""": datasets.Value("""string"""						,    id="""sequence"""    ),
														        }    )						,    codebase_urls=["""https://github.com/krishnap25/mauve"""]						,    reference_urls=[
														        """https://arxiv.org/abs/2102.01454""",
														        """https://github.com/krishnap25/mauve""",
														    ]						,    )



							def 						lowerCAmelCase    (			self							: Optional[int]						,    UpperCAmelCase_							: Tuple						,    UpperCAmelCase_							: Any						,    UpperCAmelCase_							: Optional[Any]=None						,    UpperCAmelCase_							: Optional[Any]=None						,    UpperCAmelCase_							: List[Any]=None						,    UpperCAmelCase_							: Any=None						,    UpperCAmelCase_							: int="auto"						,    UpperCAmelCase_							: Dict=-1						,    UpperCAmelCase_							: Optional[Any]=0.9						,    UpperCAmelCase_							: Dict=5						,    UpperCAmelCase_							: Optional[int]=500						,    UpperCAmelCase_							: Any="gpt2-large"						,    UpperCAmelCase_							: Union[str, Any]=-1						,    UpperCAmelCase_							: Optional[Any]=1_024						,    UpperCAmelCase_							: Dict=25						,    UpperCAmelCase_							: Optional[Any]=5						,    UpperCAmelCase_							: int=True						,    UpperCAmelCase_							: Union[str, Any]=25						,    )   ->List[Any]:



														"""simple docstring"""



														snake_case_          =						compute_mauve(
														    p_text=UpperCAmelCase_						,    q_text=UpperCAmelCase_						,    p_features=UpperCAmelCase_						,    q_features=UpperCAmelCase_						,    p_tokens=UpperCAmelCase_						,    q_tokens=UpperCAmelCase_						,    num_buckets=UpperCAmelCase_						,    pca_max_data=UpperCAmelCase_						,    kmeans_explained_var=UpperCAmelCase_						,    kmeans_num_redo=UpperCAmelCase_						,    kmeans_max_iter=UpperCAmelCase_						,    featurize_model_name=UpperCAmelCase_						,    device_id=UpperCAmelCase_						,    max_text_length=UpperCAmelCase_						,    divergence_curve_discretization_size=UpperCAmelCase_						,    mauve_scaling_factor=UpperCAmelCase_						,    verbose=UpperCAmelCase_						,    seed=UpperCAmelCase_						,    )
														return out



