turing/tests/behavioral/test_invariance.py

import pytest

# These tests check that "noise" (like capitalization or punctuation) does not change the prediction


@pytest.mark.parametrize(
    "comment",
    [
        ":param user_id: The ID of the user.",  # Base
        ":PARAM USER_ID: THE ID OF THE USER.",  # Uppercase
        "  :param user_id: The ID of the  user .  ",  # Whitespace
        ":param user_id: The ID of the user!!!",  # Punctuation
    ],
)
def test_python_invariance_parameters(python_model, comment, get_predicted_labels):
    """Tests that noise  doesn't break ':param' detection."""
    expected = {"Parameters"}
    preds = get_predicted_labels(python_model, comment, "python")
    assert preds == expected


def test_java_invariance_deprecation(java_model, get_predicted_labels):
    """Tests that noise doesn't break '@deprecated' detection"""
    base_comment = "/** @deprecated Use newUserMethod() */"
    pert_comment = "/** @DEPRECATED... Use newUserMethod()!!! */"

    preds_base = get_predicted_labels(java_model, base_comment, "java")
    preds_pert = get_predicted_labels(java_model, pert_comment, "java")

    assert {"deprecation"} <= preds_base
    assert preds_base == preds_pert


def test_python_invariance_summary(python_model, get_predicted_labels):
    """Tests that noise doesn't break a simple 'Summary' detection"""

    base_comment = "a service specific account of type bar."
    expected = {"Summary"}

    # Perturbations
    variants = [
        base_comment,
        "A SERVICE SPECIFIC ACCOUNT OF TYPE BAR.",
        "  a service specific account of type bar.  ",
        "a service specific account of type bar!!!",
    ]

    for comment in variants:
        preds = get_predicted_labels(python_model, comment, "python")
        assert preds == expected


def test_pharo_invariance_intent(pharo_model, get_predicted_labels):
    """Tests that noise doesn't break Pharo's 'Intent' detection"""

    base_comment = '"i am a simple arrow like arrowhead."'
    expected = {"Intent"}

    # Perturbations
    variants = [
        base_comment,
        '"I AM A SIMPLE ARROW LIKE ARROWHEAD."',
        '  "i am a simple arrow like arrowhead."  ',
        '"i am a simple arrow like arrowhead !!"',  #
    ]

    for comment in variants:
        preds = get_predicted_labels(pharo_model, comment, "pharo")
        assert preds == expected


def test_python_invariance_typos_parameters(python_model, get_predicted_labels):
    """
    Tests typo tolerance

    """

    # Define the single expected outcome
    expected_labels = {"Parameters"}

    # Define the base case and all its variants (with typos)
    variants = [
        ":param user_id: The ID of the user.",
        ":paramater user_id: The ID of the user.",
        ":pram user_id: The ID of teh user.",
    ]

    # Loop through all variants and assert they all produce the *exact* expected outcome
    for comment in variants:
        preds = get_predicted_labels(python_model, comment, "python")
        assert preds == expected_labels


def test_java_invariance_semantic_summary(java_model, get_predicted_labels):
    """
    Tests semantic invariance

    """

    # Get the prediction for the base comment
    base_comment = "/** Returns the user ID. */"
    base_preds = get_predicted_labels(java_model, base_comment, "java")

    # Define semantic paraphrases of the base comment
    variants = [
        base_comment,
        "/** Gets the user ID. */",
        "/** Fetches the ID for the user. */",
        "/** A method to return the user's ID. */",
    ]

    # Check that the base prediction is valid (summary)
    assert "summary" in base_preds

    for comment in variants:
        preds = get_predicted_labels(java_model, comment, "java")
        assert preds == base_preds