test pytest plugin

Cover validate_text assertions, fixture factories, marker registration,
and pytest integration using pytester for subprocess testing.

tests/test_pytest_plugin/test_assertions.py

@@ -0,0 +1,171 @@
+"""Tests for the validate_text assertion function."""
+
+import pytest
+
+from veritext.pytest_plugin import validate_text
+
+
+class TestValidateTextBasicValidation:
+    def test_valid_length(self) -> None:
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(text, min_length=10, max_length=100)
+
+    def test_too_short(self) -> None:
+        text = "Short."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, min_length=50)
+        assert "length" in str(exc_info.value).lower()
+
+    def test_too_long(self) -> None:
+        text = "A" * 100
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, max_length=50)
+        assert "length" in str(exc_info.value).lower()
+
+
+class TestValidateTextReadability:
+    def test_simple_text_passes(self) -> None:
+        text = "The cat sat on the mat. It was a nice day."
+        validate_text(text, max_reading_grade=10.0)
+
+    def test_complex_text_fails(self) -> None:
+        text = (
+            "The implementation of sophisticated metacognitive strategies "
+            "necessitates the thorough understanding of epistemological "
+            "frameworks and their corresponding methodological implications."
+        )
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, max_reading_grade=3.0)
+        assert "readability" in str(exc_info.value).lower()
+
+
+class TestValidateTextPatterns:
+    def test_contains_pattern(self) -> None:
+        text = "Please contact support@example.com for assistance."
+        validate_text(text, must_contain=["support@example.com"])
+
+    def test_missing_pattern(self) -> None:
+        text = "Please contact us for assistance."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, must_contain=["@example.com"])
+        assert "contains" in str(exc_info.value).lower()
+
+    def test_excludes_pattern(self) -> None:
+        text = "The report is complete and reviewed."
+        validate_text(text, must_exclude=["TODO", "FIXME"])
+
+    def test_forbidden_pattern(self) -> None:
+        text = "The report is almost done. TODO: add conclusion."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, must_exclude=["TODO"])
+        assert "excludes" in str(exc_info.value).lower()
+
+
+class TestValidateTextComparisonMetrics:
+    def test_high_bleu_passes(self) -> None:
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(text, reference=reference, min_bleu=0.9)
+
+    def test_low_bleu_fails(self) -> None:
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "A slow red cat sleeps under the active mouse."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, reference=reference, min_bleu=0.5)
+        assert "bleu" in str(exc_info.value).lower()
+
+    def test_high_rouge_passes(self) -> None:
+        reference = "Machine learning models require extensive training data."
+        text = "Machine learning models need extensive training data."
+        validate_text(text, reference=reference, min_rouge=0.5)
+
+    def test_low_rouge_fails(self) -> None:
+        reference = "The algorithm processes input data efficiently."
+        text = "Cats enjoy sleeping in sunny spots."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, reference=reference, min_rouge=0.5)
+        assert "rouge" in str(exc_info.value).lower()
+
+
+class TestValidateTextErrorHandling:
+    def test_requires_criteria(self) -> None:
+        with pytest.raises(ValueError, match="At least one validation criterion"):
+            validate_text("Some text")
+
+    def test_bleu_requires_reference(self) -> None:
+        with pytest.raises(ValueError, match="Reference text required"):
+            validate_text("Some text", min_bleu=0.5)
+
+    def test_rouge_requires_reference(self) -> None:
+        with pytest.raises(ValueError, match="Reference text required"):
+            validate_text("Some text", min_rouge=0.5)
+
+    def test_semantic_requires_reference(self) -> None:
+        with pytest.raises(ValueError, match="Reference text required"):
+            validate_text("Some text", min_semantic=0.5)
+
+
+class TestValidateTextMultipleCriteria:
+    def test_all_criteria_pass(self) -> None:
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(
+            text,
+            reference=reference,
+            min_bleu=0.9,
+            min_length=10,
+            max_length=100,
+        )
+
+    def test_one_failure_fails_all(self) -> None:
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "The quick brown fox jumps over the lazy dog."
+        with pytest.raises(AssertionError):
+            validate_text(
+                text,
+                reference=reference,
+                min_bleu=0.9,
+                max_length=10,  # This will fail
+            )
+
+
+class TestValidateTextFailureMessage:
+    def test_includes_text_preview(self) -> None:
+        text = "Short text"
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, min_length=100)
+        assert "Short text" in str(exc_info.value)
+
+    def test_truncates_long_text(self) -> None:
+        text = "A" * 200
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, max_length=50)
+        message = str(exc_info.value)
+        assert "..." in message
+        assert "A" * 200 not in message
+
+    def test_includes_check_details(self) -> None:
+        text = "Short"
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, min_length=100)
+        message = str(exc_info.value)
+        assert "Failed checks:" in message
+        assert "length" in message.lower()
+
+
+class TestValidateTextListReference:
+    def test_bleu_multi_reference(self) -> None:
+        references = [
+            "The quick brown fox jumps over the lazy dog.",
+            "A fast brown fox leaps over a sleepy dog.",
+        ]
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(text, reference=references, min_bleu=0.9)
+
+    def test_rouge_multi_reference(self) -> None:
+        references = [
+            "Machine learning requires data.",
+            "ML models need training data.",
+        ]
+        text = "Machine learning models require training data."
+        validate_text(text, reference=references, min_rouge=0.3)