test(validators): add validator tests
Add comprehensive tests for metric validators, constraint validators, and composite validators covering pass/fail cases and error handling.
This commit is contained in:
1
tests/test_validators/__init__.py
Normal file
1
tests/test_validators/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Tests for the validators module."""
|
||||
198
tests/test_validators/test_composite.py
Normal file
198
tests/test_validators/test_composite.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""Tests for composite validators."""
|
||||
|
||||
import pytest
|
||||
|
||||
from veritext.core.types import ValidationContext
|
||||
from veritext.validators import all_of, any_of, bleu, contains, excludes, length
|
||||
from veritext.validators.composite import AllOf, AnyOf
|
||||
|
||||
|
||||
class TestAllOf:
|
||||
"""Tests for AllOf composite validator."""
|
||||
|
||||
def test_all_of_passes_when_all_checks_pass(self) -> None:
|
||||
"""Test that AllOf passes when all checks pass."""
|
||||
validator = AllOf(
|
||||
checks=[
|
||||
length(min_words=2),
|
||||
contains(patterns=["hello"]),
|
||||
]
|
||||
)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert len(result.checks) == 2
|
||||
assert all(c.passed for c in result.checks)
|
||||
|
||||
def test_all_of_fails_when_one_check_fails(self) -> None:
|
||||
"""Test that AllOf fails when any check fails."""
|
||||
validator = AllOf(
|
||||
checks=[
|
||||
length(min_words=2),
|
||||
contains(patterns=["goodbye"]),
|
||||
]
|
||||
)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert len(result.checks) == 2
|
||||
assert len(result.failed_checks) == 1
|
||||
|
||||
def test_all_of_fails_when_all_checks_fail(self) -> None:
|
||||
"""Test that AllOf fails when all checks fail."""
|
||||
validator = AllOf(
|
||||
checks=[
|
||||
length(min_words=10),
|
||||
contains(patterns=["goodbye"]),
|
||||
]
|
||||
)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert len(result.failed_checks) == 2
|
||||
|
||||
def test_all_of_with_metric_validators(self) -> None:
|
||||
"""Test AllOf with metric-based validators."""
|
||||
validator = AllOf(
|
||||
checks=[
|
||||
bleu(min_score=0.5),
|
||||
length(min_words=3),
|
||||
]
|
||||
)
|
||||
context = ValidationContext(reference="the quick brown fox")
|
||||
result = validator.check("the quick brown fox jumps", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert len(result.checks) == 2
|
||||
|
||||
def test_all_of_failure_summary(self) -> None:
|
||||
"""Test the failure summary property."""
|
||||
validator = AllOf(
|
||||
checks=[
|
||||
length(min_words=10),
|
||||
contains(patterns=["goodbye"]),
|
||||
]
|
||||
)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello", context)
|
||||
|
||||
summary = result.failure_summary
|
||||
assert "failed" in summary.lower()
|
||||
assert "length" in summary
|
||||
assert "contains" in summary
|
||||
|
||||
def test_all_of_raises_on_empty_checks(self) -> None:
|
||||
"""Test that empty checks list raises error."""
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
AllOf(checks=[])
|
||||
|
||||
def test_all_of_name_property(self) -> None:
|
||||
"""Test the name property."""
|
||||
validator = AllOf(checks=[length(min_chars=1)])
|
||||
assert validator.name == "all_of"
|
||||
|
||||
def test_all_of_factory_function(self) -> None:
|
||||
"""Test the all_of() factory function."""
|
||||
validator = all_of(checks=[length(min_chars=1)])
|
||||
assert isinstance(validator, AllOf)
|
||||
|
||||
|
||||
class TestAnyOf:
|
||||
"""Tests for AnyOf composite validator."""
|
||||
|
||||
def test_any_of_passes_when_any_check_passes(self) -> None:
|
||||
"""Test that AnyOf passes when any check passes."""
|
||||
validator = AnyOf(
|
||||
checks=[
|
||||
length(min_words=10), # Will fail
|
||||
contains(patterns=["hello"]), # Will pass
|
||||
]
|
||||
)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert len(result.checks) == 2
|
||||
# At least one check passed
|
||||
assert any(c.passed for c in result.checks)
|
||||
|
||||
def test_any_of_passes_when_all_checks_pass(self) -> None:
|
||||
"""Test that AnyOf passes when all checks pass."""
|
||||
validator = AnyOf(
|
||||
checks=[
|
||||
length(min_words=2),
|
||||
contains(patterns=["hello"]),
|
||||
]
|
||||
)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert all(c.passed for c in result.checks)
|
||||
|
||||
def test_any_of_fails_when_all_checks_fail(self) -> None:
|
||||
"""Test that AnyOf fails when all checks fail."""
|
||||
validator = AnyOf(
|
||||
checks=[
|
||||
length(min_words=10),
|
||||
contains(patterns=["goodbye"]),
|
||||
]
|
||||
)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert not any(c.passed for c in result.checks)
|
||||
|
||||
def test_any_of_with_metric_validators(self) -> None:
|
||||
"""Test AnyOf with metric-based validators."""
|
||||
validator = AnyOf(
|
||||
checks=[
|
||||
bleu(min_score=0.9), # Might fail
|
||||
length(min_words=3), # Should pass
|
||||
]
|
||||
)
|
||||
context = ValidationContext(reference="different text entirely")
|
||||
result = validator.check("the quick brown fox jumps", context)
|
||||
|
||||
assert result.passed is True # Length check passes
|
||||
|
||||
def test_any_of_with_excludes(self) -> None:
|
||||
"""Test AnyOf with excludes validator."""
|
||||
validator = AnyOf(
|
||||
checks=[
|
||||
excludes(patterns=["error"]),
|
||||
excludes(patterns=["warning"]),
|
||||
]
|
||||
)
|
||||
context = ValidationContext()
|
||||
|
||||
# Should pass - neither pattern found
|
||||
result = validator.check("All is well", context)
|
||||
assert result.passed is True
|
||||
|
||||
# Should pass - one pattern found, other not
|
||||
result = validator.check("This is an error", context)
|
||||
assert result.passed is True
|
||||
|
||||
# Should fail - both patterns found
|
||||
result = validator.check("error and warning", context)
|
||||
assert result.passed is False
|
||||
|
||||
def test_any_of_raises_on_empty_checks(self) -> None:
|
||||
"""Test that empty checks list raises error."""
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
AnyOf(checks=[])
|
||||
|
||||
def test_any_of_name_property(self) -> None:
|
||||
"""Test the name property."""
|
||||
validator = AnyOf(checks=[length(min_chars=1)])
|
||||
assert validator.name == "any_of"
|
||||
|
||||
def test_any_of_factory_function(self) -> None:
|
||||
"""Test the any_of() factory function."""
|
||||
validator = any_of(checks=[length(min_chars=1)])
|
||||
assert isinstance(validator, AnyOf)
|
||||
334
tests/test_validators/test_constraint.py
Normal file
334
tests/test_validators/test_constraint.py
Normal file
@@ -0,0 +1,334 @@
|
||||
"""Tests for constraint validators."""
|
||||
|
||||
import pytest
|
||||
|
||||
from veritext.core.exceptions import InvalidThresholdError
|
||||
from veritext.core.types import ValidationContext
|
||||
from veritext.validators import contains, excludes, length, readability
|
||||
from veritext.validators.constraint import (
|
||||
ContainsValidator,
|
||||
ExcludesValidator,
|
||||
LengthValidator,
|
||||
ReadabilityValidator,
|
||||
)
|
||||
|
||||
|
||||
class TestLengthValidator:
|
||||
"""Tests for LengthValidator."""
|
||||
|
||||
def test_length_validator_min_chars_passes(self) -> None:
|
||||
"""Test that validator passes when char count meets minimum."""
|
||||
validator = LengthValidator(min_chars=10)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world!", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.name == "length"
|
||||
assert result.actual["chars"] == 12
|
||||
|
||||
def test_length_validator_min_chars_fails(self) -> None:
|
||||
"""Test that validator fails when char count below minimum."""
|
||||
validator = LengthValidator(min_chars=20)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert "< min" in result.message
|
||||
|
||||
def test_length_validator_max_chars_passes(self) -> None:
|
||||
"""Test that validator passes when char count within maximum."""
|
||||
validator = LengthValidator(max_chars=20)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.actual["chars"] == 11
|
||||
|
||||
def test_length_validator_max_chars_fails(self) -> None:
|
||||
"""Test that validator fails when char count exceeds maximum."""
|
||||
validator = LengthValidator(max_chars=5)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert "> max" in result.message
|
||||
|
||||
def test_length_validator_min_words_passes(self) -> None:
|
||||
"""Test that validator passes when word count meets minimum."""
|
||||
validator = LengthValidator(min_words=3)
|
||||
context = ValidationContext()
|
||||
result = validator.check("the quick brown fox", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.actual["words"] == 4
|
||||
|
||||
def test_length_validator_min_words_fails(self) -> None:
|
||||
"""Test that validator fails when word count below minimum."""
|
||||
validator = LengthValidator(min_words=10)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert "words < min" in result.message
|
||||
|
||||
def test_length_validator_max_words_passes(self) -> None:
|
||||
"""Test that validator passes when word count within maximum."""
|
||||
validator = LengthValidator(max_words=5)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world", context)
|
||||
|
||||
assert result.passed is True
|
||||
|
||||
def test_length_validator_max_words_fails(self) -> None:
|
||||
"""Test that validator fails when word count exceeds maximum."""
|
||||
validator = LengthValidator(max_words=2)
|
||||
context = ValidationContext()
|
||||
result = validator.check("the quick brown fox", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert "words > max" in result.message
|
||||
|
||||
def test_length_validator_combined_constraints(self) -> None:
|
||||
"""Test validator with multiple constraints."""
|
||||
validator = LengthValidator(
|
||||
min_chars=5, max_chars=50, min_words=2, max_words=10
|
||||
)
|
||||
context = ValidationContext()
|
||||
result = validator.check("the quick brown fox", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert "min_chars" in result.threshold
|
||||
assert "max_chars" in result.threshold
|
||||
assert "min_words" in result.threshold
|
||||
assert "max_words" in result.threshold
|
||||
|
||||
def test_length_validator_raises_when_no_constraints(self) -> None:
|
||||
"""Test that validator raises when no constraints provided."""
|
||||
with pytest.raises(InvalidThresholdError, match="At least one"):
|
||||
LengthValidator()
|
||||
|
||||
def test_length_validator_raises_on_negative_values(self) -> None:
|
||||
"""Test that negative constraint values raise error."""
|
||||
with pytest.raises(InvalidThresholdError, match="min_chars must be >= 0"):
|
||||
LengthValidator(min_chars=-1)
|
||||
|
||||
with pytest.raises(InvalidThresholdError, match="max_chars must be >= 0"):
|
||||
LengthValidator(max_chars=-1)
|
||||
|
||||
with pytest.raises(InvalidThresholdError, match="min_words must be >= 0"):
|
||||
LengthValidator(min_words=-1)
|
||||
|
||||
with pytest.raises(InvalidThresholdError, match="max_words must be >= 0"):
|
||||
LengthValidator(max_words=-1)
|
||||
|
||||
def test_length_validator_raises_on_invalid_range(self) -> None:
|
||||
"""Test that min > max raises error."""
|
||||
with pytest.raises(InvalidThresholdError, match="cannot exceed max_chars"):
|
||||
LengthValidator(min_chars=100, max_chars=50)
|
||||
|
||||
with pytest.raises(InvalidThresholdError, match="cannot exceed max_words"):
|
||||
LengthValidator(min_words=20, max_words=5)
|
||||
|
||||
def test_length_factory_function(self) -> None:
|
||||
"""Test the length() factory function."""
|
||||
validator = length(min_chars=10, max_words=100)
|
||||
assert isinstance(validator, LengthValidator)
|
||||
assert validator.name == "length"
|
||||
|
||||
|
||||
class TestReadabilityValidator:
|
||||
"""Tests for ReadabilityValidator."""
|
||||
|
||||
def test_readability_validator_max_grade_passes(self) -> None:
|
||||
"""Test that validator passes when grade level within maximum."""
|
||||
validator = ReadabilityValidator(max_grade=12.0)
|
||||
context = ValidationContext()
|
||||
# Simple text should have low grade level
|
||||
result = validator.check("The cat sat on the mat. It was a nice day.", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.name == "readability"
|
||||
assert "grade" in result.actual
|
||||
|
||||
def test_readability_validator_max_grade_fails(self) -> None:
|
||||
"""Test that validator fails when grade level exceeds maximum."""
|
||||
validator = ReadabilityValidator(max_grade=1.0)
|
||||
context = ValidationContext()
|
||||
# Complex text
|
||||
result = validator.check(
|
||||
"The implementation of sophisticated methodologies necessitates "
|
||||
"comprehensive analytical frameworks for systematic evaluation.",
|
||||
context,
|
||||
)
|
||||
|
||||
assert result.passed is False
|
||||
assert "grade level" in result.message
|
||||
assert "> max" in result.message
|
||||
|
||||
def test_readability_validator_min_ease_passes(self) -> None:
|
||||
"""Test that validator passes when reading ease meets minimum."""
|
||||
validator = ReadabilityValidator(min_ease=30.0)
|
||||
context = ValidationContext()
|
||||
# Simple text should have high reading ease
|
||||
result = validator.check("The cat sat. The dog ran. It was fun.", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert "ease" in result.actual
|
||||
|
||||
def test_readability_validator_min_ease_fails(self) -> None:
|
||||
"""Test that validator fails when reading ease below minimum."""
|
||||
validator = ReadabilityValidator(min_ease=100.0)
|
||||
context = ValidationContext()
|
||||
result = validator.check(
|
||||
"The implementation of sophisticated methodologies necessitates "
|
||||
"comprehensive analytical frameworks.",
|
||||
context,
|
||||
)
|
||||
|
||||
assert result.passed is False
|
||||
assert "reading ease" in result.message
|
||||
assert "< min" in result.message
|
||||
|
||||
def test_readability_validator_combined_constraints(self) -> None:
|
||||
"""Test validator with both grade and ease constraints."""
|
||||
validator = ReadabilityValidator(max_grade=12.0, min_ease=30.0)
|
||||
context = ValidationContext()
|
||||
result = validator.check("The cat sat on the mat.", context)
|
||||
|
||||
assert "max_grade" in result.threshold
|
||||
assert "min_ease" in result.threshold
|
||||
|
||||
def test_readability_validator_raises_when_no_constraints(self) -> None:
|
||||
"""Test that validator raises when no constraints provided."""
|
||||
with pytest.raises(InvalidThresholdError, match="At least one"):
|
||||
ReadabilityValidator()
|
||||
|
||||
def test_readability_factory_function(self) -> None:
|
||||
"""Test the readability() factory function."""
|
||||
validator = readability(max_grade=8.0, min_ease=60.0)
|
||||
assert isinstance(validator, ReadabilityValidator)
|
||||
assert validator.name == "readability"
|
||||
|
||||
|
||||
class TestContainsValidator:
|
||||
"""Tests for ContainsValidator."""
|
||||
|
||||
def test_contains_validator_passes_when_pattern_found(self) -> None:
|
||||
"""Test that validator passes when all patterns are found."""
|
||||
validator = ContainsValidator(patterns=["hello", "world"])
|
||||
context = ValidationContext()
|
||||
result = validator.check("Hello World!", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.name == "contains"
|
||||
assert result.actual["found"] == 2
|
||||
assert result.actual["missing"] == []
|
||||
|
||||
def test_contains_validator_fails_when_pattern_missing(self) -> None:
|
||||
"""Test that validator fails when a pattern is missing."""
|
||||
validator = ContainsValidator(patterns=["hello", "goodbye"])
|
||||
context = ValidationContext()
|
||||
result = validator.check("Hello World!", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert "goodbye" in result.actual["missing"]
|
||||
assert "missing" in result.message
|
||||
|
||||
def test_contains_validator_case_insensitive_by_default(self) -> None:
|
||||
"""Test that matching is case-insensitive by default."""
|
||||
validator = ContainsValidator(patterns=["HELLO"])
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world", context)
|
||||
|
||||
assert result.passed is True
|
||||
|
||||
def test_contains_validator_case_sensitive(self) -> None:
|
||||
"""Test case-sensitive matching."""
|
||||
validator = ContainsValidator(patterns=["HELLO"], case_sensitive=True)
|
||||
context = ValidationContext()
|
||||
result = validator.check("hello world", context)
|
||||
|
||||
assert result.passed is False
|
||||
|
||||
def test_contains_validator_regex_patterns(self) -> None:
|
||||
"""Test regex pattern matching."""
|
||||
validator = ContainsValidator(patterns=[r"\d{3}-\d{4}"])
|
||||
context = ValidationContext()
|
||||
result = validator.check("Call 555-1234 for info", context)
|
||||
|
||||
assert result.passed is True
|
||||
|
||||
def test_contains_validator_raises_on_empty_patterns(self) -> None:
|
||||
"""Test that empty patterns list raises error."""
|
||||
with pytest.raises(InvalidThresholdError, match="cannot be empty"):
|
||||
ContainsValidator(patterns=[])
|
||||
|
||||
def test_contains_factory_function(self) -> None:
|
||||
"""Test the contains() factory function."""
|
||||
validator = contains(patterns=["test"], case_sensitive=True)
|
||||
assert isinstance(validator, ContainsValidator)
|
||||
assert validator.name == "contains"
|
||||
|
||||
|
||||
class TestExcludesValidator:
|
||||
"""Tests for ExcludesValidator."""
|
||||
|
||||
def test_excludes_validator_passes_when_pattern_absent(self) -> None:
|
||||
"""Test that validator passes when all patterns are absent."""
|
||||
validator = ExcludesValidator(patterns=["bad", "forbidden"])
|
||||
context = ValidationContext()
|
||||
result = validator.check("This is good text.", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.name == "excludes"
|
||||
assert result.actual["found"] == []
|
||||
|
||||
def test_excludes_validator_fails_when_pattern_found(self) -> None:
|
||||
"""Test that validator fails when a forbidden pattern is found."""
|
||||
validator = ExcludesValidator(patterns=["bad", "forbidden"])
|
||||
context = ValidationContext()
|
||||
result = validator.check("This is bad text.", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert "bad" in result.actual["found"]
|
||||
assert "forbidden" in result.message
|
||||
|
||||
def test_excludes_validator_case_insensitive_by_default(self) -> None:
|
||||
"""Test that matching is case-insensitive by default."""
|
||||
validator = ExcludesValidator(patterns=["BAD"])
|
||||
context = ValidationContext()
|
||||
result = validator.check("This is bad text.", context)
|
||||
|
||||
assert result.passed is False
|
||||
|
||||
def test_excludes_validator_case_sensitive(self) -> None:
|
||||
"""Test case-sensitive matching."""
|
||||
validator = ExcludesValidator(patterns=["BAD"], case_sensitive=True)
|
||||
context = ValidationContext()
|
||||
result = validator.check("This is bad text.", context)
|
||||
|
||||
assert result.passed is True
|
||||
|
||||
def test_excludes_validator_regex_patterns(self) -> None:
|
||||
"""Test regex pattern matching."""
|
||||
validator = ExcludesValidator(patterns=[r"\b\d{4}\b"]) # 4-digit numbers
|
||||
context = ValidationContext()
|
||||
|
||||
# Should fail when pattern found
|
||||
result = validator.check("PIN is 1234", context)
|
||||
assert result.passed is False
|
||||
|
||||
# Should pass when pattern absent
|
||||
result = validator.check("No numbers here", context)
|
||||
assert result.passed is True
|
||||
|
||||
def test_excludes_validator_raises_on_empty_patterns(self) -> None:
|
||||
"""Test that empty patterns list raises error."""
|
||||
with pytest.raises(InvalidThresholdError, match="cannot be empty"):
|
||||
ExcludesValidator(patterns=[])
|
||||
|
||||
def test_excludes_factory_function(self) -> None:
|
||||
"""Test the excludes() factory function."""
|
||||
validator = excludes(patterns=["test"], case_sensitive=True)
|
||||
assert isinstance(validator, ExcludesValidator)
|
||||
assert validator.name == "excludes"
|
||||
209
tests/test_validators/test_metric.py
Normal file
209
tests/test_validators/test_metric.py
Normal file
@@ -0,0 +1,209 @@
|
||||
"""Tests for metric-based validators."""
|
||||
|
||||
import pytest
|
||||
|
||||
from veritext.core.exceptions import InvalidThresholdError, ValidationError
|
||||
from veritext.core.types import ValidationContext
|
||||
from veritext.validators import bleu, lexical, rouge
|
||||
from veritext.validators.metric import BleuValidator, LexicalValidator, RougeValidator
|
||||
|
||||
|
||||
class TestBleuValidator:
|
||||
"""Tests for BleuValidator."""
|
||||
|
||||
def test_bleu_validator_passes_when_score_meets_threshold(self) -> None:
|
||||
"""Test that validator passes when BLEU score meets threshold."""
|
||||
validator = BleuValidator(min_score=0.5, variant=4)
|
||||
context = ValidationContext(reference="the cat sat on the mat")
|
||||
result = validator.check("the cat sat on the mat", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.name == "bleu-4"
|
||||
assert result.actual == 1.0 # Identical text
|
||||
assert result.threshold == 0.5
|
||||
|
||||
def test_bleu_validator_fails_when_score_below_threshold(self) -> None:
|
||||
"""Test that validator fails when BLEU score is below threshold."""
|
||||
validator = BleuValidator(min_score=0.9, variant=4)
|
||||
context = ValidationContext(reference="the cat sat on the mat")
|
||||
result = validator.check("a dog ran through the park", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert result.name == "bleu-4"
|
||||
assert result.actual < 0.9
|
||||
assert "below minimum" in result.message
|
||||
|
||||
def test_bleu_validator_variant_selection(self) -> None:
|
||||
"""Test different BLEU variants."""
|
||||
context = ValidationContext(reference="the quick brown fox jumps")
|
||||
|
||||
for variant in (1, 2, 3, 4):
|
||||
validator = BleuValidator(min_score=0.0, variant=variant) # type: ignore[arg-type]
|
||||
result = validator.check("the quick brown fox", context)
|
||||
assert result.name == f"bleu-{variant}"
|
||||
|
||||
def test_bleu_validator_raises_on_missing_reference(self) -> None:
|
||||
"""Test that validator raises when reference is missing."""
|
||||
validator = BleuValidator(min_score=0.5)
|
||||
context = ValidationContext()
|
||||
|
||||
with pytest.raises(ValidationError, match="requires reference text"):
|
||||
validator.check("some text", context)
|
||||
|
||||
def test_bleu_validator_raises_on_invalid_min_score(self) -> None:
|
||||
"""Test that invalid min_score raises error."""
|
||||
with pytest.raises(InvalidThresholdError, match=r"between 0\.0 and 1\.0"):
|
||||
BleuValidator(min_score=1.5)
|
||||
|
||||
with pytest.raises(InvalidThresholdError, match=r"between 0\.0 and 1\.0"):
|
||||
BleuValidator(min_score=-0.1)
|
||||
|
||||
def test_bleu_validator_raises_on_invalid_variant(self) -> None:
|
||||
"""Test that invalid variant raises error."""
|
||||
with pytest.raises(InvalidThresholdError, match="variant must be"):
|
||||
BleuValidator(min_score=0.5, variant=5) # type: ignore[arg-type]
|
||||
|
||||
def test_bleu_factory_function(self) -> None:
|
||||
"""Test the bleu() factory function."""
|
||||
validator = bleu(min_score=0.6, variant=2)
|
||||
assert isinstance(validator, BleuValidator)
|
||||
assert validator.name == "bleu-2"
|
||||
|
||||
|
||||
class TestRougeValidator:
|
||||
"""Tests for RougeValidator."""
|
||||
|
||||
def test_rouge_validator_passes_when_score_meets_threshold(self) -> None:
|
||||
"""Test that validator passes when ROUGE score meets threshold."""
|
||||
validator = RougeValidator(min_score=0.5, variant="l")
|
||||
context = ValidationContext(reference="the cat sat on the mat")
|
||||
result = validator.check("the cat sat on the mat", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.name == "rouge-l"
|
||||
assert result.actual == 1.0 # Identical text
|
||||
assert result.threshold == 0.5
|
||||
|
||||
def test_rouge_validator_fails_when_score_below_threshold(self) -> None:
|
||||
"""Test that validator fails when ROUGE score is below threshold."""
|
||||
validator = RougeValidator(min_score=0.9, variant="l")
|
||||
context = ValidationContext(reference="the cat sat on the mat")
|
||||
result = validator.check("a dog ran through the park", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert result.actual < 0.9
|
||||
assert "below minimum" in result.message
|
||||
|
||||
def test_rouge_validator_variant_selection(self) -> None:
|
||||
"""Test different ROUGE variants."""
|
||||
context = ValidationContext(reference="the quick brown fox jumps")
|
||||
|
||||
for variant in ("1", "2", "l"):
|
||||
validator = RougeValidator(min_score=0.0, variant=variant) # type: ignore[arg-type]
|
||||
result = validator.check("the quick brown fox", context)
|
||||
assert result.name == f"rouge-{variant}"
|
||||
|
||||
def test_rouge_validator_raises_on_missing_reference(self) -> None:
|
||||
"""Test that validator raises when reference is missing."""
|
||||
validator = RougeValidator(min_score=0.5)
|
||||
context = ValidationContext()
|
||||
|
||||
with pytest.raises(ValidationError, match="requires reference text"):
|
||||
validator.check("some text", context)
|
||||
|
||||
def test_rouge_validator_raises_on_invalid_min_score(self) -> None:
|
||||
"""Test that invalid min_score raises error."""
|
||||
with pytest.raises(InvalidThresholdError, match=r"between 0\.0 and 1\.0"):
|
||||
RougeValidator(min_score=1.5)
|
||||
|
||||
def test_rouge_validator_raises_on_invalid_variant(self) -> None:
|
||||
"""Test that invalid variant raises error."""
|
||||
with pytest.raises(InvalidThresholdError, match="variant must be"):
|
||||
RougeValidator(min_score=0.5, variant="3") # type: ignore[arg-type]
|
||||
|
||||
def test_rouge_factory_function(self) -> None:
|
||||
"""Test the rouge() factory function."""
|
||||
validator = rouge(min_score=0.6, variant="2")
|
||||
assert isinstance(validator, RougeValidator)
|
||||
assert validator.name == "rouge-2"
|
||||
|
||||
|
||||
class TestLexicalValidator:
|
||||
"""Tests for LexicalValidator."""
|
||||
|
||||
def test_lexical_validator_passes_on_jaccard(self) -> None:
|
||||
"""Test that validator passes when Jaccard similarity meets threshold."""
|
||||
validator = LexicalValidator(min_jaccard=0.5)
|
||||
context = ValidationContext(reference="the cat sat on the mat")
|
||||
result = validator.check("the cat sat on the mat", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.name == "lexical"
|
||||
assert result.actual["jaccard"] == 1.0
|
||||
|
||||
def test_lexical_validator_fails_on_jaccard(self) -> None:
|
||||
"""Test that validator fails when Jaccard is below threshold."""
|
||||
validator = LexicalValidator(min_jaccard=0.9)
|
||||
context = ValidationContext(reference="the cat sat on the mat")
|
||||
result = validator.check("a dog ran through the park", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert "Jaccard" in result.message
|
||||
assert "below minimum" in result.message
|
||||
|
||||
def test_lexical_validator_passes_on_overlap(self) -> None:
|
||||
"""Test that validator passes when token overlap meets threshold."""
|
||||
validator = LexicalValidator(min_overlap=0.5)
|
||||
context = ValidationContext(reference="the cat sat on the mat")
|
||||
result = validator.check("the cat sat on the mat", context)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.actual["token_overlap"] == 1.0
|
||||
|
||||
def test_lexical_validator_fails_on_overlap(self) -> None:
|
||||
"""Test that validator fails when overlap is below threshold."""
|
||||
validator = LexicalValidator(min_overlap=0.9)
|
||||
context = ValidationContext(reference="the cat sat on the mat")
|
||||
result = validator.check("a dog ran through", context)
|
||||
|
||||
assert result.passed is False
|
||||
assert "overlap" in result.message
|
||||
|
||||
def test_lexical_validator_with_both_thresholds(self) -> None:
|
||||
"""Test validator with both Jaccard and overlap thresholds."""
|
||||
validator = LexicalValidator(min_jaccard=0.3, min_overlap=0.5)
|
||||
context = ValidationContext(reference="the cat sat on the mat")
|
||||
result = validator.check("the cat sat", context)
|
||||
|
||||
# Should check both thresholds
|
||||
assert "min_jaccard" in result.threshold
|
||||
assert "min_overlap" in result.threshold
|
||||
|
||||
def test_lexical_validator_raises_when_no_threshold(self) -> None:
|
||||
"""Test that validator raises when no threshold is provided."""
|
||||
with pytest.raises(InvalidThresholdError, match="At least one"):
|
||||
LexicalValidator()
|
||||
|
||||
def test_lexical_validator_raises_on_invalid_jaccard(self) -> None:
|
||||
"""Test that invalid Jaccard threshold raises error."""
|
||||
with pytest.raises(InvalidThresholdError, match="min_jaccard"):
|
||||
LexicalValidator(min_jaccard=1.5)
|
||||
|
||||
def test_lexical_validator_raises_on_invalid_overlap(self) -> None:
|
||||
"""Test that invalid overlap threshold raises error."""
|
||||
with pytest.raises(InvalidThresholdError, match="min_overlap"):
|
||||
LexicalValidator(min_overlap=-0.1)
|
||||
|
||||
def test_lexical_validator_raises_on_missing_reference(self) -> None:
|
||||
"""Test that validator raises when reference is missing."""
|
||||
validator = LexicalValidator(min_jaccard=0.5)
|
||||
context = ValidationContext()
|
||||
|
||||
with pytest.raises(ValidationError, match="requires reference text"):
|
||||
validator.check("some text", context)
|
||||
|
||||
def test_lexical_factory_function(self) -> None:
|
||||
"""Test the lexical() factory function."""
|
||||
validator = lexical(min_jaccard=0.5, min_overlap=0.6)
|
||||
assert isinstance(validator, LexicalValidator)
|
||||
assert validator.name == "lexical"
|
||||
Reference in New Issue
Block a user