

'''simple docstring'''


import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
				import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case				:				Optional[int]       							=		logging.get_logger(__name__)

_snake_case				:				Tuple       							=		{'vocab_file': 'vocab.txt'}

_snake_case				:				List[str]       							=		{
    'vocab_file': {
        'openbmb/cpm-ant-10b': 'https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt',
    },
}

_snake_case				:				Union[str, Any]       							=		{
    'openbmb/cpm-ant-10b': 1024,
}
def 		snake_case_   (UpperCamelCase	:  List[Any]				):





							'''simple docstring'''


							_a     					=    collections.OrderedDict()
							with open(lowerCamelCase__			,						'''r'''			,						encoding='''utf-8'''				) as reader:
														_a     					=    reader.readlines()
							for index, token in enumerate(lowerCamelCase__				):
														_a     					=    token.rstrip('''\n'''				)
														_a     					=    index
							return vocab

class       A   (		snake_case_   ):







							def __init__(  self	:  List[Any]							,      lowerCAmelCase_	:  int							,      lowerCAmelCase_	:  Optional[Any]="<unk>"							,      lowerCAmelCase_	:  Union[str, Any]=2_00 )      ->							str:


														"""simple docstring"""


														_a     					=    vocab
														_a     					=    unk_token
														_a     					=    max_input_chars_per_word







							def        __lowerCAmelCase      (  self	:  int							,      lowerCAmelCase_	:  Dict )      ->							str:


														"""simple docstring"""


														_a     					=    list(lowerCAmelCase_ )
														if len(lowerCAmelCase_ ) > self.max_input_chars_per_word:
																					return [self.unk_token]

														_a     					=    0
														_a     					=    []
														while start < len(lowerCAmelCase_ ):
																					_a     					=    len(lowerCAmelCase_ )
																					_a     					=    None
																					while start < end:
																												_a     					=    ''''''.join(chars[start:end] )
																												if substr in self.vocab:
																																			_a     					=    substr
																																			break
																												end -= 1
																					if cur_substr is None:
																												sub_tokens.append(self.unk_token )
																												start += 1
																					else:
																												sub_tokens.append(lowerCAmelCase_ )
																												_a     					=    end

														return sub_tokens



class       A   (		snake_case_   ):
							lowercase_          =    VOCAB_FILES_NAMES
							lowercase_          =    PRETRAINED_VOCAB_FILES_MAP
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_          =    ['input_ids', 'attention_mask']
							lowercase_          =    False







							def __init__(  self	:  Union[str, Any]							,      lowerCAmelCase_	:  Tuple							,      lowerCAmelCase_	:  str="<d>"							,      lowerCAmelCase_	:  List[Any]="</d>"							,      lowerCAmelCase_	:  Tuple="<s>"							,      lowerCAmelCase_	:  Optional[Any]="</s>"							,      lowerCAmelCase_	:  Dict="<pad>"							,      lowerCAmelCase_	:  Optional[Any]="<unk>"							,      lowerCAmelCase_	:  int="</n>"							,      lowerCAmelCase_	:  str="</_>"							,      lowerCAmelCase_	:  Any="left"							,      **lowerCAmelCase_	:  Dict							,      )      ->							List[Any]:


														"""simple docstring"""


														requires_backends(self							,      ['''jieba'''] )
														super().__init__(
														    bod_token=lowerCAmelCase_							,      eod_token=lowerCAmelCase_							,      bos_token=lowerCAmelCase_							,      eos_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      unk_token=lowerCAmelCase_							,      line_token=lowerCAmelCase_							,      space_token=lowerCAmelCase_							,      padding_side=lowerCAmelCase_							,      **lowerCAmelCase_							,      )
														_a     					=    bod_token
														_a     					=    eod_token
														_a     					=    load_vocab(lowerCAmelCase_ )
														_a     					=    self.encoder[space_token]
														_a     					=    self.encoder[line_token]

														del self.encoder[space_token]
														del self.encoder[line_token]

														_a     					=    collections.OrderedDict(sorted(self.encoder.items()							,      key=lambda lowerCAmelCase_ : x[1] ) )
														_a     					=    {v: k for k, v in self.encoder.items()}

														_a     					=    WordpieceTokenizer(vocab=self.encoder							,      unk_token=self.unk_token )







							@property
							def        __lowerCAmelCase      (  self	:  Tuple )      ->							Tuple:


														"""simple docstring"""


														return self.encoder[self.bod_token]







							@property
							def        __lowerCAmelCase      (  self	:  Tuple )      ->							Optional[int]:


														"""simple docstring"""


														return self.encoder[self.eod_token]







							@property
							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							Optional[int]:


														"""simple docstring"""


														return self.encoder["\n"]







							@property
							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							int:


														"""simple docstring"""


														return len(self.encoder )







							def        __lowerCAmelCase      (  self	:  int )      ->							Dict:


														"""simple docstring"""


														return dict(self.encoder							,      **self.added_tokens_encoder )







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  Tuple )      ->							Optional[int]:


														"""simple docstring"""


														_a     					=    []
														for x in jieba.cut(lowerCAmelCase_							,      cut_all=lowerCAmelCase_ ):
																					output_tokens.extend(self.wordpiece_tokenizer.tokenize(lowerCAmelCase_ ) )
														return output_tokens







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  List[str]							,      **lowerCAmelCase_	:  Any )      ->							Tuple:


														"""simple docstring"""


														_a     					=    [i for i in token_ids if i >= 0]
														_a     					=    [
														    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
														]
														return super()._decode(lowerCAmelCase_							,      **lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  int							,      lowerCAmelCase_	:  Optional[Any] )      ->							Union[str, Any]:


														"""simple docstring"""


														return token in self.encoder







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  int )      ->							str:


														"""simple docstring"""


														return "".join(lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Optional[Any]							,      lowerCAmelCase_	:  Tuple )      ->							Dict:


														"""simple docstring"""


														return self.encoder.get(lowerCAmelCase_							,      self.encoder.get(self.unk_token ) )







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  Union[str, Any] )      ->							Optional[Any]:


														"""simple docstring"""


														return self.decoder.get(lowerCAmelCase_							,      self.unk_token )







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      lowerCAmelCase_	:  Dict							,      lowerCAmelCase_	:  Optional[Any] = None )      ->							Tuple[str]:


														"""simple docstring"""


														if os.path.isdir(lowerCAmelCase_ ):
																					_a     					=    os.path.join(
																					    lowerCAmelCase_							,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
														else:
																					_a     					=    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
														_a     					=    0
														if " " in self.encoder:
																					_a     					=    self.encoder[''' ''']
																					del self.encoder[" "]
														if "\n" in self.encoder:
																					_a     					=    self.encoder['''\n''']
																					del self.encoder["\n"]
														_a     					=    collections.OrderedDict(sorted(self.encoder.items()							,      key=lambda lowerCAmelCase_ : x[1] ) )
														with open(lowerCAmelCase_							,      '''w'''							,      encoding='''utf-8''' ) as writer:
																					for token, token_index in self.encoder.items():
																												if index != token_index:
																																			logger.warning(
																																			    F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																																			    ''' Please check that the vocabulary is not corrupted!''' )
																																			_a     					=    token_index
																												writer.write(token + '''\n''' )
																												index += 1
														return (vocab_file,)







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  Tuple							,      lowerCAmelCase_	:  Optional[int] = None )      ->							List[int]:


														"""simple docstring"""


														if token_ids_a is None:
																					return [self.bos_token_id] + token_ids_a
														return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a







							def        __lowerCAmelCase      (  self	:  Dict							,      lowerCAmelCase_	:  List[str]							,      lowerCAmelCase_	:  Tuple = None							,      lowerCAmelCase_	:  int = False )      ->							List[int]:


														"""simple docstring"""


														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=lowerCAmelCase_							,      token_ids_a=lowerCAmelCase_							,      already_has_special_tokens=lowerCAmelCase_ )

														if token_ids_a is not None:
																					return [1] + ([0] * len(lowerCAmelCase_ )) + [1] + ([0] * len(lowerCAmelCase_ ))
														return [1] + ([0] * len(lowerCAmelCase_ ))







'''simple docstring'''


from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

import numpy as np
import PIL
from PIL import Image

from ...utils import BaseOutput, is_torch_available, is_transformers_available

@dataclass
class       A   (		_a   ):
							lowercase_          =    42
							lowercase_          =    42


if is_transformers_available() and is_torch_available():
				from .pipeline_semantic_stable_diffusion import SemanticStableDiffusionPipeline

