

import re
import tempfile
from pathlib import Path

import pytest
import yaml

from datasets.utils.readme import ReadMe


# @pytest.fixture
# def example_yaml_structure():

snake_case_        =			yaml.safe_load(
    '''\
name: ""
allow_empty: false
allow_empty_text: true
subsections:
  - name: "Dataset Card for X" # First-level markdown heading
    allow_empty: false
    allow_empty_text: true
    subsections:
      - name: "Table of Contents"
        allow_empty: false
        allow_empty_text: false
        subsections: null
      - name: "Dataset Description"
        allow_empty: false
        allow_empty_text: false
        subsections:
          - name: "Dataset Summary"
            allow_empty: false
            allow_empty_text: false
            subsections: null
          - name: "Supported Tasks and Leaderboards"
            allow_empty: true
            allow_empty_text: true
            subsections: null
          - name: Languages
            allow_empty: false
            allow_empty_text: true
            subsections: null
'''
)


snake_case_        =			{
    '''name''': '''root''',
    '''text''': '''''',
    '''is_empty_text''': True,
    '''subsections''': [
        {
            '''name''': '''Dataset Card for My Dataset''',
            '''text''': '''''',
            '''is_empty_text''': True,
            '''subsections''': [
                {'''name''': '''Table of Contents''', '''text''': '''Some text here.''', '''is_empty_text''': False, '''subsections''': []},
                {
                    '''name''': '''Dataset Description''',
                    '''text''': '''Some text here.''',
                    '''is_empty_text''': False,
                    '''subsections''': [
                        {
                            '''name''': '''Dataset Summary''',
                            '''text''': '''Some text here.''',
                            '''is_empty_text''': False,
                            '''subsections''': [],
                        },
                        {
                            '''name''': '''Supported Tasks and Leaderboards''',
                            '''text''': '''''',
                            '''is_empty_text''': True,
                            '''subsections''': [],
                        },
                        {'''name''': '''Languages''', '''text''': '''Language Text''', '''is_empty_text''': False, '''subsections''': []},
                    ],
                },
            ],
        }
    ],
}


snake_case_        =			'''\
---
language:
- zh
- en
---

# Dataset Card for My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
### Supported Tasks and Leaderboards
### Languages
Language Text
'''


snake_case_        =			'''\
---
language:
- zh
- en
---

# Dataset Card for My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
#### Extra Ignored Subsection
### Supported Tasks and Leaderboards
### Languages
Language Text
'''

snake_case_        =			{
    '''name''': '''root''',
    '''text''': '''''',
    '''is_empty_text''': True,
    '''subsections''': [
        {
            '''name''': '''Dataset Card for My Dataset''',
            '''text''': '''''',
            '''is_empty_text''': True,
            '''subsections''': [
                {'''name''': '''Table of Contents''', '''text''': '''Some text here.''', '''is_empty_text''': False, '''subsections''': []},
                {
                    '''name''': '''Dataset Description''',
                    '''text''': '''Some text here.''',
                    '''is_empty_text''': False,
                    '''subsections''': [
                        {
                            '''name''': '''Dataset Summary''',
                            '''text''': '''Some text here.''',
                            '''is_empty_text''': False,
                            '''subsections''': [
                                {
                                    '''name''': '''Extra Ignored Subsection''',
                                    '''text''': '''''',
                                    '''is_empty_text''': True,
                                    '''subsections''': [],
                                }
                            ],
                        },
                        {
                            '''name''': '''Supported Tasks and Leaderboards''',
                            '''text''': '''''',
                            '''is_empty_text''': True,
                            '''subsections''': [],
                        },
                        {'''name''': '''Languages''', '''text''': '''Language Text''', '''is_empty_text''': False, '''subsections''': []},
                    ],
                },
            ],
        }
    ],
}

snake_case_        =			'''\
---
---
# Dataset Card for My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
### Supported Tasks and Leaderboards
### Languages
Language Text
'''

snake_case_        =			(
    '''The following issues were found for the README at `{path}`:\n-\tEmpty YAML markers are present in the README.'''
)

snake_case_        =			'''\
# Dataset Card for My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
### Supported Tasks and Leaderboards
### Languages
Language Text
'''

snake_case_        =			(
    '''The following issues were found for the README at `{path}`:\n-\tNo YAML markers are present in the README.'''
)

snake_case_        =			'''\
---
# Dataset Card for My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
### Supported Tasks and Leaderboards
### Languages
Language Text
'''

snake_case_        =			'''The following issues were found for the README at `{path}`:\n-\tOnly the start of YAML tags present in the README.'''

snake_case_        =			'''\
---
language:
- zh
- en
---

# Dataset Card for My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
### Supported Tasks and Leaderboards
### Languages
Language Text
'''
snake_case_        =			'''The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Summary` but it is empty.\n-\tExpected some text in section `Dataset Summary` but it is empty (text in subsections are ignored).'''


snake_case_        =			'''\
---
language:
- zh
- en
---

# Dataset Card for My Dataset
'''
snake_case_        =			'''The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Card for My Dataset` but it is empty.\n-\tSection `Dataset Card for My Dataset` expected the following subsections: `Table of Contents`, `Dataset Description`. Found \'None\'.'''

snake_case_        =			'''\
---
language:
- zh
- en
---

# Dataset Card for My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
### Languages
Language Text
'''

snake_case_        =			'''The following issues were found for the README at `{path}`:\n-\tSection `Dataset Description` is missing subsection: `Supported Tasks and Leaderboards`.'''


snake_case_        =			'''\
---
language:
- zh
- en
---

# Dataset Card for My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
### Supported Tasks and Leaderboards
### Languages
'''

snake_case_        =			'''The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Languages` but it is empty.'''

snake_case_        =			'''\
---
language:
- zh
- en
---

## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
### Supported Tasks and Leaderboards
### Languages
Language Text
'''
snake_case_        =			'''The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.'''

snake_case_        =			'''\
---
language:
- zh
- en
---

# Dataset Card for My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
### Supported Tasks and Leaderboards
### Languages
Language Text
# Dataset Card My Dataset
'''

snake_case_        =			'''The following issues were found for the README at `{path}`:\n-\tThe README has several first-level headings: `Dataset Card for My Dataset`, `Dataset Card My Dataset`. Only one heading is expected. Skipping further validation for this README.'''

snake_case_        =			'''\
---
language:
- zh
- en
---

# Dataset Card My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
### Supported Tasks and Leaderboards
### Languages
Language Text
'''

snake_case_        =			'''The following issues were found for the README at `{path}`:\n-\tNo first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'''

snake_case_        =			''''''

snake_case_        =			'''The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.\n-\tNo YAML markers are present in the README.'''

snake_case_        =			'''\
---
language:
- zh
- en
---

# Dataset Card for My Dataset
# Dataset Card for My Dataset
## Table of Contents
Some text here.
## Dataset Description
Some text here.
### Dataset Summary
Some text here.
### Supported Tasks and Leaderboards
### Languages
Language Text
'''

snake_case_        =			'''The following issues were found while parsing the README at `{path}`:\n-\tMultiple sections with the same heading `Dataset Card for My Dataset` have been found. Please keep only one of these sections.'''

@pytest.mark.parametrize(
    'readme_md, expected_dict'     ,					[
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]     ,					)
def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   Union[str, Any]     ,					SCREAMING_SNAKE_CASE_		:   str      ):



     '''simple docstring'''





     assert ReadMe.from_string(SCREAMING_SNAKE_CASE_     ,					SCREAMING_SNAKE_CASE_      ).to_dict() == expected_dict

@pytest.mark.parametrize(
    'readme_md, expected_error'     ,					[
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]     ,					)
def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   Optional[int]     ,					SCREAMING_SNAKE_CASE_		:   Dict      ):



     '''simple docstring'''





     with pytest.raises(SCREAMING_SNAKE_CASE_     ,					match=re.escape(expected_error.format(path='root'      )      )      ):
          lowercase__   :       Tuple				=	ReadMe.from_string(SCREAMING_SNAKE_CASE_     ,					SCREAMING_SNAKE_CASE_      )
          readme.validate()

@pytest.mark.parametrize(
    'readme_md, expected_error'     ,					[
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]     ,					)
def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   Tuple     ,					SCREAMING_SNAKE_CASE_		:   Optional[Any]      ):



     '''simple docstring'''





     with pytest.raises(SCREAMING_SNAKE_CASE_     ,					match=re.escape(expected_error.format(path='root'      )      )      ):
          ReadMe.from_string(SCREAMING_SNAKE_CASE_     ,					SCREAMING_SNAKE_CASE_      )

@pytest.mark.parametrize(
    'readme_md,'     ,					[
        (README_MULTIPLE_SAME_HEADING_1),
    ]     ,					)
def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   Optional[Any]      ):



     '''simple docstring'''





     ReadMe.from_string(SCREAMING_SNAKE_CASE_     ,					SCREAMING_SNAKE_CASE_     ,					suppress_parsing_errors=SCREAMING_SNAKE_CASE_      )

@pytest.mark.parametrize(
    'readme_md, expected_dict'     ,					[
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]     ,					)
def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   Optional[int]     ,					SCREAMING_SNAKE_CASE_		:   str      ):



     '''simple docstring'''





     with tempfile.TemporaryDirectory() as tmp_dir:
          lowercase__   :       Union[str, Any]				=	Path(SCREAMING_SNAKE_CASE_      ) / 'README.md'
          with open(SCREAMING_SNAKE_CASE_     ,					'w+'      ) as readme_file:
               readme_file.write(SCREAMING_SNAKE_CASE_      )
          lowercase__   :       Any				=	ReadMe.from_readme(SCREAMING_SNAKE_CASE_     ,					SCREAMING_SNAKE_CASE_      ).to_dict()
          assert out["name"] == path
          assert out["text"] == ""
          assert out["is_empty_text"]
          assert out["subsections"] == expected_dict["subsections"]

@pytest.mark.parametrize(
    'readme_md, expected_error'     ,					[
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]     ,					)
def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   List[Any]     ,					SCREAMING_SNAKE_CASE_		:   str      ):



     '''simple docstring'''





     with tempfile.TemporaryDirectory() as tmp_dir:
          lowercase__   :       str				=	Path(SCREAMING_SNAKE_CASE_      ) / 'README.md'
          with open(SCREAMING_SNAKE_CASE_     ,					'w+'      ) as readme_file:
               readme_file.write(SCREAMING_SNAKE_CASE_      )
          lowercase__   :       str				=	expected_error.format(path=SCREAMING_SNAKE_CASE_      )
          with pytest.raises(SCREAMING_SNAKE_CASE_     ,					match=re.escape(SCREAMING_SNAKE_CASE_      )      ):
               lowercase__   :       List[str]				=	ReadMe.from_readme(SCREAMING_SNAKE_CASE_     ,					SCREAMING_SNAKE_CASE_      )
               readme.validate()

@pytest.mark.parametrize(
    'readme_md, expected_error'     ,					[
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]     ,					)
def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   Optional[int]     ,					SCREAMING_SNAKE_CASE_		:   str      ):



     '''simple docstring'''





     with tempfile.TemporaryDirectory() as tmp_dir:
          lowercase__   :       Union[str, Any]				=	Path(SCREAMING_SNAKE_CASE_      ) / 'README.md'
          with open(SCREAMING_SNAKE_CASE_     ,					'w+'      ) as readme_file:
               readme_file.write(SCREAMING_SNAKE_CASE_      )
          lowercase__   :       List[Any]				=	expected_error.format(path=SCREAMING_SNAKE_CASE_      )
          with pytest.raises(SCREAMING_SNAKE_CASE_     ,					match=re.escape(SCREAMING_SNAKE_CASE_      )      ):
               ReadMe.from_readme(SCREAMING_SNAKE_CASE_     ,					SCREAMING_SNAKE_CASE_      )

@pytest.mark.parametrize(
    'readme_md,'     ,					[
        (README_MULTIPLE_SAME_HEADING_1),
    ]     ,					)
def      snake_case__      (   SCREAMING_SNAKE_CASE_		:   Tuple      ):



     '''simple docstring'''





     with tempfile.TemporaryDirectory() as tmp_dir:
          lowercase__   :       Union[str, Any]				=	Path(SCREAMING_SNAKE_CASE_      ) / 'README.md'
          with open(SCREAMING_SNAKE_CASE_     ,					'w+'      ) as readme_file:
               readme_file.write(SCREAMING_SNAKE_CASE_      )
          ReadMe.from_readme(SCREAMING_SNAKE_CASE_     ,					SCREAMING_SNAKE_CASE_     ,					suppress_parsing_errors=SCREAMING_SNAKE_CASE_      )




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


snake_case_        =			logging.get_logger(__name__)

snake_case_        =			{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

snake_case_        =			{
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

snake_case_        =			{
    '''junnyu/roformer_chinese_small''': 1_536,
    '''junnyu/roformer_chinese_base''': 1_536,
    '''junnyu/roformer_chinese_char_small''': 512,
    '''junnyu/roformer_chinese_char_base''': 512,
    '''junnyu/roformer_small_discriminator''': 128,
    '''junnyu/roformer_small_generator''': 128,
}


snake_case_        =			{
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}


class 				SCREAMING_SNAKE_CASE__ (__snake_case					):
    __lowerCamelCase						:	Union[str, Any]														=					VOCAB_FILES_NAMES
    __lowerCamelCase						:	int														=					PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase						:	Any														=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowerCamelCase						:	int														=					PRETRAINED_INIT_CONFIGURATION
    __lowerCamelCase						:	Tuple														=					RoFormerTokenizer





    def __init__(				self						,       a=None						,       a=None						,       a=True						,       a="[UNK]"						,       a="[SEP]"						,       a="[PAD]"						,       a="[CLS]"						,       a="[MASK]"						,       a=True						,       a=None						,       **a						,       ):
         super().__init__(
             a						,       tokenizer_file=a						,       do_lower_case=a						,       unk_token=a						,       sep_token=a						,       pad_token=a						,       cls_token=a						,       mask_token=a						,       tokenize_chinese_chars=a						,       strip_accents=a						,       **a						,       )

         lowercase__   :       Optional[Any]				=	json.loads(self.backend_tokenizer.normalizer.__getstate__())
         if (
             pre_tok_state.get('lowercase'						,       a) != do_lower_case
             or pre_tok_state.get('strip_accents'						,       a) != strip_accents
         ):
              lowercase__   :       Optional[int]				=	getattr(a						,       pre_tok_state.pop('type'))
              lowercase__   :       str				=	do_lower_case
              lowercase__   :       Union[str, Any]				=	strip_accents
              lowercase__   :       int				=	pre_tok_class(**a)

         lowercase__   :       Optional[int]				=	do_lower_case





    def __getstate__(				self):
         lowercase__   :       str				=	self.__dict__.copy()
         lowercase__   :       Any				=	BertPreTokenizer()
         return state





    def __setstate__(				self						,       a):
         lowercase__   :       Union[str, Any]				=	d
         lowercase__   :       int				=	self.__dict__['_tokenizer'].get_vocab()
         lowercase__   :       List[str]				=	PreTokenizer.custom(JiebaPreTokenizer(a))





    def 				snake_case_    (				self						,       a						,       a=None):
         lowercase__   :       Union[str, Any]				=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

         return output





    def 				snake_case_    (				self						,       a						,       a = None):
         lowercase__   :       Tuple				=	[self.sep_token_id]
         lowercase__   :       Optional[int]				=	[self.cls_token_id]
         if token_ids_a is None:
              return len(cls + token_ids_a + sep) * [0]
         return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





    def 				snake_case_    (				self						,       a						,       a = None):
         lowercase__   :       Optional[Any]				=	self._tokenizer.model.save(a						,       name=a)
         return tuple(a)





    def 				snake_case_    (				self						,       a						,       a=None						,       a=None						,       a=False						,       **a						,       ):
         lowercase__   :       List[str]				=	BertPreTokenizer()
         return super().save_pretrained(a						,       a						,       a						,       a						,       **a)

