From 7f2358640625be152829c1e563568d238b78fd93 Mon Sep 17 00:00:00 2001 From: Kai Chappell Date: Sat, 29 Mar 2025 13:30:23 +0000 Subject: [PATCH] validator tests Add comprehensive tests for metric validators, constraint validators, and composite validators covering pass/fail cases and error handling. --- tests/test_validators/__init__.py | 1 + tests/test_validators/test_composite.py | 178 ++++++++++++++ tests/test_validators/test_constraint.py | 300 +++++++++++++++++++++++ tests/test_validators/test_metric.py | 244 ++++++++++++++++++ 4 files changed, 723 insertions(+) create mode 100644 tests/test_validators/__init__.py create mode 100644 tests/test_validators/test_composite.py create mode 100644 tests/test_validators/test_constraint.py create mode 100644 tests/test_validators/test_metric.py diff --git a/tests/test_validators/__init__.py b/tests/test_validators/__init__.py new file mode 100644 index 0000000..2375d84 --- /dev/null +++ b/tests/test_validators/__init__.py @@ -0,0 +1 @@ +"""Tests for the validators module.""" diff --git a/tests/test_validators/test_composite.py b/tests/test_validators/test_composite.py new file mode 100644 index 0000000..c4b7a3a --- /dev/null +++ b/tests/test_validators/test_composite.py @@ -0,0 +1,178 @@ +"""Tests for composite validators.""" + +import pytest + +from veritext.core.types import ValidationContext +from veritext.validators import all_of, any_of, bleu, contains, excludes, length +from veritext.validators.composite import AllOf, AnyOf + + +class TestAllOf: + def test_all_of_passes_when_all_checks_pass(self) -> None: + validator = AllOf( + checks=[ + length(min_words=2), + contains(patterns=["hello"]), + ] + ) + context = ValidationContext() + result = validator.check("hello world", context) + + assert result.passed is True + assert len(result.checks) == 2 + assert all(c.passed for c in result.checks) + + def test_all_of_fails_when_one_check_fails(self) -> None: + validator = AllOf( + checks=[ + length(min_words=2), + contains(patterns=["goodbye"]), + ] + ) + context = ValidationContext() + result = validator.check("hello world", context) + + assert result.passed is False + assert len(result.checks) == 2 + assert len(result.failed_checks) == 1 + + def test_all_of_fails_when_all_checks_fail(self) -> None: + validator = AllOf( + checks=[ + length(min_words=10), + contains(patterns=["goodbye"]), + ] + ) + context = ValidationContext() + result = validator.check("hello", context) + + assert result.passed is False + assert len(result.failed_checks) == 2 + + def test_all_of_with_metric_validators(self) -> None: + validator = AllOf( + checks=[ + bleu(min_score=0.5), + length(min_words=3), + ] + ) + context = ValidationContext(reference="the quick brown fox") + result = validator.check("the quick brown fox jumps", context) + + assert result.passed is True + assert len(result.checks) == 2 + + def test_all_of_failure_summary(self) -> None: + validator = AllOf( + checks=[ + length(min_words=10), + contains(patterns=["goodbye"]), + ] + ) + context = ValidationContext() + result = validator.check("hello", context) + + summary = result.failure_summary + assert "failed" in summary.lower() + assert "length" in summary + assert "contains" in summary + + def test_all_of_raises_on_empty_checks(self) -> None: + with pytest.raises(ValueError, match="cannot be empty"): + AllOf(checks=[]) + + def test_all_of_name_property(self) -> None: + validator = AllOf(checks=[length(min_chars=1)]) + assert validator.name == "all_of" + + def test_all_of_factory_function(self) -> None: + validator = all_of(checks=[length(min_chars=1)]) + assert isinstance(validator, AllOf) + + +class TestAnyOf: + def test_any_of_passes_when_any_check_passes(self) -> None: + validator = AnyOf( + checks=[ + length(min_words=10), # Will fail + contains(patterns=["hello"]), # Will pass + ] + ) + context = ValidationContext() + result = validator.check("hello world", context) + + assert result.passed is True + assert len(result.checks) == 2 + # At least one check passed + assert any(c.passed for c in result.checks) + + def test_any_of_passes_when_all_checks_pass(self) -> None: + validator = AnyOf( + checks=[ + length(min_words=2), + contains(patterns=["hello"]), + ] + ) + context = ValidationContext() + result = validator.check("hello world", context) + + assert result.passed is True + assert all(c.passed for c in result.checks) + + def test_any_of_fails_when_all_checks_fail(self) -> None: + validator = AnyOf( + checks=[ + length(min_words=10), + contains(patterns=["goodbye"]), + ] + ) + context = ValidationContext() + result = validator.check("hello", context) + + assert result.passed is False + assert not any(c.passed for c in result.checks) + + def test_any_of_with_metric_validators(self) -> None: + validator = AnyOf( + checks=[ + bleu(min_score=0.9), # Might fail + length(min_words=3), # Should pass + ] + ) + context = ValidationContext(reference="different text entirely") + result = validator.check("the quick brown fox jumps", context) + + assert result.passed is True # Length check passes + + def test_any_of_with_excludes(self) -> None: + validator = AnyOf( + checks=[ + excludes(patterns=["error"]), + excludes(patterns=["warning"]), + ] + ) + context = ValidationContext() + + # Should pass - neither pattern found + result = validator.check("All is well", context) + assert result.passed is True + + # Should pass - one pattern found, other not + result = validator.check("This is an error", context) + assert result.passed is True + + # Should fail - both patterns found + result = validator.check("error and warning", context) + assert result.passed is False + + def test_any_of_raises_on_empty_checks(self) -> None: + with pytest.raises(ValueError, match="cannot be empty"): + AnyOf(checks=[]) + + def test_any_of_name_property(self) -> None: + validator = AnyOf(checks=[length(min_chars=1)]) + assert validator.name == "any_of" + + def test_any_of_factory_function(self) -> None: + validator = any_of(checks=[length(min_chars=1)]) + assert isinstance(validator, AnyOf) diff --git a/tests/test_validators/test_constraint.py b/tests/test_validators/test_constraint.py new file mode 100644 index 0000000..899e594 --- /dev/null +++ b/tests/test_validators/test_constraint.py @@ -0,0 +1,300 @@ +"""Tests for constraint validators.""" + +import pytest + +from veritext.core.exceptions import InvalidThresholdError +from veritext.core.types import ValidationContext +from veritext.validators import contains, excludes, length, readability +from veritext.validators.constraint import ( + ContainsValidator, + ExcludesValidator, + LengthValidator, + ReadabilityValidator, +) + + +class TestLengthValidator: + def test_min_chars_passes(self) -> None: + validator = LengthValidator(min_chars=10) + context = ValidationContext() + result = validator.check("hello world!", context) + + assert result.passed is True + assert result.name == "length" + assert result.actual["chars"] == 12 + + def test_min_chars_fails(self) -> None: + validator = LengthValidator(min_chars=20) + context = ValidationContext() + result = validator.check("hello", context) + + assert result.passed is False + assert "< min" in result.message + + def test_max_chars_passes(self) -> None: + validator = LengthValidator(max_chars=20) + context = ValidationContext() + result = validator.check("hello world", context) + + assert result.passed is True + assert result.actual["chars"] == 11 + + def test_max_chars_fails(self) -> None: + validator = LengthValidator(max_chars=5) + context = ValidationContext() + result = validator.check("hello world", context) + + assert result.passed is False + assert "> max" in result.message + + def test_min_words_passes(self) -> None: + validator = LengthValidator(min_words=3) + context = ValidationContext() + result = validator.check("the quick brown fox", context) + + assert result.passed is True + assert result.actual["words"] == 4 + + def test_min_words_fails(self) -> None: + validator = LengthValidator(min_words=10) + context = ValidationContext() + result = validator.check("hello world", context) + + assert result.passed is False + assert "words < min" in result.message + + def test_max_words_passes(self) -> None: + validator = LengthValidator(max_words=5) + context = ValidationContext() + result = validator.check("hello world", context) + + assert result.passed is True + + def test_max_words_fails(self) -> None: + validator = LengthValidator(max_words=2) + context = ValidationContext() + result = validator.check("the quick brown fox", context) + + assert result.passed is False + assert "words > max" in result.message + + def test_combined_constraints(self) -> None: + validator = LengthValidator( + min_chars=5, max_chars=50, min_words=2, max_words=10 + ) + context = ValidationContext() + result = validator.check("the quick brown fox", context) + + assert result.passed is True + assert "min_chars" in result.threshold + assert "max_chars" in result.threshold + assert "min_words" in result.threshold + assert "max_words" in result.threshold + + def test_requires_constraints(self) -> None: + with pytest.raises(InvalidThresholdError, match="At least one"): + LengthValidator() + + def test_rejects_negative_values(self) -> None: + with pytest.raises(InvalidThresholdError, match="min_chars must be >= 0"): + LengthValidator(min_chars=-1) + + with pytest.raises(InvalidThresholdError, match="max_chars must be >= 0"): + LengthValidator(max_chars=-1) + + with pytest.raises(InvalidThresholdError, match="min_words must be >= 0"): + LengthValidator(min_words=-1) + + with pytest.raises(InvalidThresholdError, match="max_words must be >= 0"): + LengthValidator(max_words=-1) + + def test_rejects_inverted_range(self) -> None: + with pytest.raises(InvalidThresholdError, match="cannot exceed max_chars"): + LengthValidator(min_chars=100, max_chars=50) + + with pytest.raises(InvalidThresholdError, match="cannot exceed max_words"): + LengthValidator(min_words=20, max_words=5) + + def test_length_factory(self) -> None: + validator = length(min_chars=10, max_words=100) + assert isinstance(validator, LengthValidator) + assert validator.name == "length" + + +class TestReadabilityValidator: + def test_max_grade_passes(self) -> None: + validator = ReadabilityValidator(max_grade=12.0) + context = ValidationContext() + # Simple text should have low grade level + result = validator.check("The cat sat on the mat. It was a nice day.", context) + + assert result.passed is True + assert result.name == "readability" + assert "grade" in result.actual + + def test_max_grade_fails(self) -> None: + validator = ReadabilityValidator(max_grade=1.0) + context = ValidationContext() + # Complex text + result = validator.check( + "The implementation of sophisticated methodologies necessitates " + "detailed analytical frameworks for systematic evaluation.", + context, + ) + + assert result.passed is False + assert "grade level" in result.message + assert "> max" in result.message + + def test_min_ease_passes(self) -> None: + validator = ReadabilityValidator(min_ease=30.0) + context = ValidationContext() + # Simple text should have high reading ease + result = validator.check("The cat sat. The dog ran. It was fun.", context) + + assert result.passed is True + assert "ease" in result.actual + + def test_min_ease_fails(self) -> None: + validator = ReadabilityValidator(min_ease=100.0) + context = ValidationContext() + result = validator.check( + "The implementation of sophisticated methodologies necessitates " + "detailed analytical frameworks.", + context, + ) + + assert result.passed is False + assert "reading ease" in result.message + assert "< min" in result.message + + def test_combined_grade_and_ease(self) -> None: + validator = ReadabilityValidator(max_grade=12.0, min_ease=30.0) + context = ValidationContext() + result = validator.check("The cat sat on the mat.", context) + + assert "max_grade" in result.threshold + assert "min_ease" in result.threshold + + def test_requires_constraints(self) -> None: + with pytest.raises(InvalidThresholdError, match="At least one"): + ReadabilityValidator() + + def test_readability_factory(self) -> None: + validator = readability(max_grade=8.0, min_ease=60.0) + assert isinstance(validator, ReadabilityValidator) + assert validator.name == "readability" + + +class TestContainsValidator: + def test_finds_all_patterns(self) -> None: + validator = ContainsValidator(patterns=["hello", "world"]) + context = ValidationContext() + result = validator.check("Hello World!", context) + + assert result.passed is True + assert result.name == "contains" + assert result.actual["found"] == 2 + assert result.actual["missing"] == [] + + def test_fails_on_missing_pattern(self) -> None: + validator = ContainsValidator(patterns=["hello", "goodbye"]) + context = ValidationContext() + result = validator.check("Hello World!", context) + + assert result.passed is False + assert "goodbye" in result.actual["missing"] + assert "missing" in result.message + + def test_case_insensitive_default(self) -> None: + validator = ContainsValidator(patterns=["HELLO"]) + context = ValidationContext() + result = validator.check("hello world", context) + + assert result.passed is True + + def test_case_sensitive(self) -> None: + validator = ContainsValidator(patterns=["HELLO"], case_sensitive=True) + context = ValidationContext() + result = validator.check("hello world", context) + + assert result.passed is False + + def test_regex_patterns(self) -> None: + validator = ContainsValidator(patterns=[r"\d{3}-\d{4}"]) + context = ValidationContext() + result = validator.check("Call 555-1234 for info", context) + + assert result.passed is True + + def test_rejects_empty_patterns(self) -> None: + with pytest.raises(InvalidThresholdError, match="cannot be empty"): + ContainsValidator(patterns=[]) + + def test_rejects_invalid_regex(self) -> None: + with pytest.raises(InvalidThresholdError, match="Invalid regex"): + ContainsValidator(patterns=[r"[invalid"]) + + def test_contains_factory(self) -> None: + validator = contains(patterns=["test"], case_sensitive=True) + assert isinstance(validator, ContainsValidator) + assert validator.name == "contains" + + +class TestExcludesValidator: + def test_passes_when_absent(self) -> None: + validator = ExcludesValidator(patterns=["bad", "forbidden"]) + context = ValidationContext() + result = validator.check("This is good text.", context) + + assert result.passed is True + assert result.name == "excludes" + assert result.actual["found"] == [] + + def test_fails_when_found(self) -> None: + validator = ExcludesValidator(patterns=["bad", "forbidden"]) + context = ValidationContext() + result = validator.check("This is bad text.", context) + + assert result.passed is False + assert "bad" in result.actual["found"] + assert "forbidden" in result.message + + def test_case_insensitive_default(self) -> None: + validator = ExcludesValidator(patterns=["BAD"]) + context = ValidationContext() + result = validator.check("This is bad text.", context) + + assert result.passed is False + + def test_case_sensitive(self) -> None: + validator = ExcludesValidator(patterns=["BAD"], case_sensitive=True) + context = ValidationContext() + result = validator.check("This is bad text.", context) + + assert result.passed is True + + def test_regex_patterns(self) -> None: + validator = ExcludesValidator(patterns=[r"\b\d{4}\b"]) # 4-digit numbers + context = ValidationContext() + + # Should fail when pattern found + result = validator.check("PIN is 1234", context) + assert result.passed is False + + # Should pass when pattern absent + result = validator.check("No numbers here", context) + assert result.passed is True + + def test_rejects_empty_patterns(self) -> None: + with pytest.raises(InvalidThresholdError, match="cannot be empty"): + ExcludesValidator(patterns=[]) + + def test_rejects_invalid_regex(self) -> None: + with pytest.raises(InvalidThresholdError, match="Invalid regex"): + ExcludesValidator(patterns=[r"[invalid"]) + + def test_excludes_factory(self) -> None: + validator = excludes(patterns=["test"], case_sensitive=True) + assert isinstance(validator, ExcludesValidator) + assert validator.name == "excludes" diff --git a/tests/test_validators/test_metric.py b/tests/test_validators/test_metric.py new file mode 100644 index 0000000..910859e --- /dev/null +++ b/tests/test_validators/test_metric.py @@ -0,0 +1,244 @@ +"""Tests for metric-based validators.""" + +import pytest + +from veritext.core.exceptions import InvalidThresholdError, ValidationError +from veritext.core.types import ValidationContext +from veritext.validators import bleu, lexical, rouge +from veritext.validators.metric import BleuValidator, LexicalValidator, RougeValidator + + +class TestBleuValidator: + def test_bleu_passes_above_threshold(self) -> None: + validator = BleuValidator(min_score=0.5, variant=4) + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check("the cat sat on the mat", context) + + assert result.passed is True + assert result.name == "bleu-4" + assert result.actual == 1.0 # Identical text + assert result.threshold == 0.5 + + def test_bleu_fails_below_threshold(self) -> None: + validator = BleuValidator(min_score=0.9, variant=4) + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check("a dog ran through the park", context) + + assert result.passed is False + assert result.name == "bleu-4" + assert result.actual < 0.9 + assert "below minimum" in result.message + + def test_bleu_variant_selection(self) -> None: + context = ValidationContext(reference="the quick brown fox jumps") + + for variant in (1, 2, 3, 4): + validator = BleuValidator(min_score=0.0, variant=variant) # type: ignore[arg-type] + result = validator.check("the quick brown fox", context) + assert result.name == f"bleu-{variant}" + + def test_bleu_requires_reference(self) -> None: + validator = BleuValidator(min_score=0.5) + context = ValidationContext() + + with pytest.raises(ValidationError, match="requires reference text"): + validator.check("some text", context) + + def test_bleu_rejects_invalid_score(self) -> None: + with pytest.raises(InvalidThresholdError, match=r"between 0\.0 and 1\.0"): + BleuValidator(min_score=1.5) + + with pytest.raises(InvalidThresholdError, match=r"between 0\.0 and 1\.0"): + BleuValidator(min_score=-0.1) + + def test_bleu_rejects_invalid_variant(self) -> None: + with pytest.raises(InvalidThresholdError, match="variant must be"): + BleuValidator(min_score=0.5, variant=5) # type: ignore[arg-type] + + def test_bleu_factory(self) -> None: + validator = bleu(min_score=0.6, variant=2) + assert isinstance(validator, BleuValidator) + assert validator.name == "bleu-2" + + +class TestRougeValidator: + def test_rouge_passes_above_threshold(self) -> None: + validator = RougeValidator(min_score=0.5, variant="l") + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check("the cat sat on the mat", context) + + assert result.passed is True + assert result.name == "rouge-l" + assert result.actual == 1.0 # Identical text + assert result.threshold == 0.5 + + def test_rouge_fails_below_threshold(self) -> None: + validator = RougeValidator(min_score=0.9, variant="l") + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check("a dog ran through the park", context) + + assert result.passed is False + assert result.actual < 0.9 + assert "below minimum" in result.message + + def test_rouge_variant_selection(self) -> None: + context = ValidationContext(reference="the quick brown fox jumps") + + for variant in ("1", "2", "l"): + validator = RougeValidator(min_score=0.0, variant=variant) # type: ignore[arg-type] + result = validator.check("the quick brown fox", context) + assert result.name == f"rouge-{variant}" + + def test_rouge_requires_reference(self) -> None: + validator = RougeValidator(min_score=0.5) + context = ValidationContext() + + with pytest.raises(ValidationError, match="requires reference text"): + validator.check("some text", context) + + def test_rouge_rejects_invalid_score(self) -> None: + with pytest.raises(InvalidThresholdError, match=r"between 0\.0 and 1\.0"): + RougeValidator(min_score=1.5) + + def test_rouge_rejects_invalid_variant(self) -> None: + with pytest.raises(InvalidThresholdError, match="variant must be"): + RougeValidator(min_score=0.5, variant="3") # type: ignore[arg-type] + + def test_rouge_factory(self) -> None: + validator = rouge(min_score=0.6, variant="2") + assert isinstance(validator, RougeValidator) + assert validator.name == "rouge-2" + + +class TestLexicalValidator: + def test_lexical_passes_jaccard(self) -> None: + validator = LexicalValidator(min_jaccard=0.5) + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check("the cat sat on the mat", context) + + assert result.passed is True + assert result.name == "lexical" + assert result.actual["jaccard"] == 1.0 + + def test_lexical_fails_jaccard(self) -> None: + validator = LexicalValidator(min_jaccard=0.9) + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check("a dog ran through the park", context) + + assert result.passed is False + assert "Jaccard" in result.message + assert "below minimum" in result.message + + def test_lexical_passes_overlap(self) -> None: + validator = LexicalValidator(min_overlap=0.5) + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check("the cat sat on the mat", context) + + assert result.passed is True + assert result.actual["token_overlap"] == 1.0 + + def test_lexical_fails_overlap(self) -> None: + validator = LexicalValidator(min_overlap=0.9) + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check("a dog ran through", context) + + assert result.passed is False + assert "overlap" in result.message + + def test_lexical_both_thresholds(self) -> None: + validator = LexicalValidator(min_jaccard=0.3, min_overlap=0.5) + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check("the cat sat", context) + + # Should check both thresholds + assert "min_jaccard" in result.threshold + assert "min_overlap" in result.threshold + + def test_lexical_needs_threshold(self) -> None: + with pytest.raises(InvalidThresholdError, match="At least one"): + LexicalValidator() + + def test_lexical_rejects_bad_jaccard(self) -> None: + with pytest.raises(InvalidThresholdError, match="min_jaccard"): + LexicalValidator(min_jaccard=1.5) + + def test_lexical_rejects_bad_overlap(self) -> None: + with pytest.raises(InvalidThresholdError, match="min_overlap"): + LexicalValidator(min_overlap=-0.1) + + def test_lexical_requires_reference(self) -> None: + validator = LexicalValidator(min_jaccard=0.5) + context = ValidationContext() + + with pytest.raises(ValidationError, match="requires reference text"): + validator.check("some text", context) + + def test_lexical_factory(self) -> None: + validator = lexical(min_jaccard=0.5, min_overlap=0.6) + assert isinstance(validator, LexicalValidator) + assert validator.name == "lexical" + + +# SemanticValidator tests - conditionally run if sentence-transformers is installed +class TestSemanticValidator: + @staticmethod + def _skip_if_no_transformers() -> None: + pytest.importorskip("sentence_transformers") + + def test_semantic_passes_above_threshold(self) -> None: + self._skip_if_no_transformers() + from veritext.validators.metric import SemanticValidator + + validator = SemanticValidator(min_score=0.5) + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check("the cat sat on the mat", context) + + assert result.passed is True + assert result.name == "semantic" + assert result.actual >= 0.99 # Identical text + assert result.threshold == 0.5 + + def test_semantic_fails_below_threshold(self) -> None: + self._skip_if_no_transformers() + from veritext.validators.metric import SemanticValidator + + validator = SemanticValidator(min_score=0.99) + context = ValidationContext(reference="the cat sat on the mat") + result = validator.check( + "quantum physics describes particle behaviour", context + ) + + assert result.passed is False + assert result.name == "semantic" + assert result.actual < 0.99 + assert "below minimum" in result.message + + def test_semantic_requires_reference(self) -> None: + self._skip_if_no_transformers() + from veritext.validators.metric import SemanticValidator + + validator = SemanticValidator(min_score=0.5) + context = ValidationContext() + + with pytest.raises(ValidationError, match="requires reference text"): + validator.check("some text", context) + + def test_semantic_rejects_invalid_score(self) -> None: + with pytest.raises(InvalidThresholdError, match=r"between 0\.0 and 1\.0"): + from veritext.validators.metric import SemanticValidator + + SemanticValidator(min_score=1.5) + + with pytest.raises(InvalidThresholdError, match=r"between 0\.0 and 1\.0"): + from veritext.validators.metric import SemanticValidator + + SemanticValidator(min_score=-0.1) + + def test_semantic_factory(self) -> None: + self._skip_if_no_transformers() + from veritext.validators import semantic + from veritext.validators.metric import SemanticValidator + + validator = semantic(min_score=0.6) + assert isinstance(validator, SemanticValidator) + assert validator.name == "semantic"