test pytest plugin

Cover validate_text assertions, fixture factories, marker registration,
and pytest integration using pytester for subprocess testing.

tests/test_pytest_plugin/__init__.py

@@ -0,0 +1 @@
+"""Tests for the Veritext pytest plugin."""

tests/test_pytest_plugin/conftest.py

@@ -0,0 +1,30 @@
+"""Pytest configuration for pytest_plugin tests."""
+
+import pytest
+
+from veritext.pytest_plugin.fixtures import ValidatorFactory
+
+# Enable the pytester fixture for plugin testing
+pytest_plugins = ["pytester"]
+
+# Re-export fixtures from the plugin module for testing
+
+
+@pytest.fixture
+def text_validator() -> ValidatorFactory:
+    return ValidatorFactory()
+
+
+@pytest.fixture
+def validation_context() -> type:
+    from typing import Any
+
+    from veritext.core.types import ValidationContext
+
+    def _create(
+        reference: str | list[str] | None = None,
+        **metadata: Any,
+    ) -> ValidationContext:
+        return ValidationContext(reference=reference, metadata=metadata)
+
+    return _create

tests/test_pytest_plugin/test_assertions.py

@@ -0,0 +1,171 @@
+"""Tests for the validate_text assertion function."""
+
+import pytest
+
+from veritext.pytest_plugin import validate_text
+
+
+class TestValidateTextBasicValidation:
+    def test_valid_length(self) -> None:
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(text, min_length=10, max_length=100)
+
+    def test_too_short(self) -> None:
+        text = "Short."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, min_length=50)
+        assert "length" in str(exc_info.value).lower()
+
+    def test_too_long(self) -> None:
+        text = "A" * 100
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, max_length=50)
+        assert "length" in str(exc_info.value).lower()
+
+
+class TestValidateTextReadability:
+    def test_simple_text_passes(self) -> None:
+        text = "The cat sat on the mat. It was a nice day."
+        validate_text(text, max_reading_grade=10.0)
+
+    def test_complex_text_fails(self) -> None:
+        text = (
+            "The implementation of sophisticated metacognitive strategies "
+            "necessitates the thorough understanding of epistemological "
+            "frameworks and their corresponding methodological implications."
+        )
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, max_reading_grade=3.0)
+        assert "readability" in str(exc_info.value).lower()
+
+
+class TestValidateTextPatterns:
+    def test_contains_pattern(self) -> None:
+        text = "Please contact support@example.com for assistance."
+        validate_text(text, must_contain=["support@example.com"])
+
+    def test_missing_pattern(self) -> None:
+        text = "Please contact us for assistance."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, must_contain=["@example.com"])
+        assert "contains" in str(exc_info.value).lower()
+
+    def test_excludes_pattern(self) -> None:
+        text = "The report is complete and reviewed."
+        validate_text(text, must_exclude=["TODO", "FIXME"])
+
+    def test_forbidden_pattern(self) -> None:
+        text = "The report is almost done. TODO: add conclusion."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, must_exclude=["TODO"])
+        assert "excludes" in str(exc_info.value).lower()
+
+
+class TestValidateTextComparisonMetrics:
+    def test_high_bleu_passes(self) -> None:
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(text, reference=reference, min_bleu=0.9)
+
+    def test_low_bleu_fails(self) -> None:
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "A slow red cat sleeps under the active mouse."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, reference=reference, min_bleu=0.5)
+        assert "bleu" in str(exc_info.value).lower()
+
+    def test_high_rouge_passes(self) -> None:
+        reference = "Machine learning models require extensive training data."
+        text = "Machine learning models need extensive training data."
+        validate_text(text, reference=reference, min_rouge=0.5)
+
+    def test_low_rouge_fails(self) -> None:
+        reference = "The algorithm processes input data efficiently."
+        text = "Cats enjoy sleeping in sunny spots."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, reference=reference, min_rouge=0.5)
+        assert "rouge" in str(exc_info.value).lower()
+
+
+class TestValidateTextErrorHandling:
+    def test_requires_criteria(self) -> None:
+        with pytest.raises(ValueError, match="At least one validation criterion"):
+            validate_text("Some text")
+
+    def test_bleu_requires_reference(self) -> None:
+        with pytest.raises(ValueError, match="Reference text required"):
+            validate_text("Some text", min_bleu=0.5)
+
+    def test_rouge_requires_reference(self) -> None:
+        with pytest.raises(ValueError, match="Reference text required"):
+            validate_text("Some text", min_rouge=0.5)
+
+    def test_semantic_requires_reference(self) -> None:
+        with pytest.raises(ValueError, match="Reference text required"):
+            validate_text("Some text", min_semantic=0.5)
+
+
+class TestValidateTextMultipleCriteria:
+    def test_all_criteria_pass(self) -> None:
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(
+            text,
+            reference=reference,
+            min_bleu=0.9,
+            min_length=10,
+            max_length=100,
+        )
+
+    def test_one_failure_fails_all(self) -> None:
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "The quick brown fox jumps over the lazy dog."
+        with pytest.raises(AssertionError):
+            validate_text(
+                text,
+                reference=reference,
+                min_bleu=0.9,
+                max_length=10,  # This will fail
+            )
+
+
+class TestValidateTextFailureMessage:
+    def test_includes_text_preview(self) -> None:
+        text = "Short text"
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, min_length=100)
+        assert "Short text" in str(exc_info.value)
+
+    def test_truncates_long_text(self) -> None:
+        text = "A" * 200
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, max_length=50)
+        message = str(exc_info.value)
+        assert "..." in message
+        assert "A" * 200 not in message
+
+    def test_includes_check_details(self) -> None:
+        text = "Short"
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, min_length=100)
+        message = str(exc_info.value)
+        assert "Failed checks:" in message
+        assert "length" in message.lower()
+
+
+class TestValidateTextListReference:
+    def test_bleu_multi_reference(self) -> None:
+        references = [
+            "The quick brown fox jumps over the lazy dog.",
+            "A fast brown fox leaps over a sleepy dog.",
+        ]
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(text, reference=references, min_bleu=0.9)
+
+    def test_rouge_multi_reference(self) -> None:
+        references = [
+            "Machine learning requires data.",
+            "ML models need training data.",
+        ]
+        text = "Machine learning models require training data."
+        validate_text(text, reference=references, min_rouge=0.3)

tests/test_pytest_plugin/test_fixtures.py

@@ -0,0 +1,74 @@
+"""Tests for the pytest plugin fixtures."""
+
+from veritext.core.types import ValidationContext
+from veritext.pytest_plugin.fixtures import ValidatorFactory
+from veritext.validators import bleu, length
+
+
+class TestValidatorFactory:
+    def test_creates_validator_from_checks(self) -> None:
+        factory = ValidatorFactory()
+        validate = factory(checks=[length(min_chars=5)])
+
+        result = validate("Hello, World!")
+        assert result.passed
+
+    def test_validator_uses_provided_reference(self) -> None:
+        factory = ValidatorFactory()
+        reference = "The quick brown fox."
+        validate = factory(
+            checks=[bleu(min_score=0.5)],
+            reference=reference,
+        )
+
+        # Exact match should pass
+        result = validate("The quick brown fox.")
+        assert result.passed
+
+    def test_validator_returns_validation_result(self) -> None:
+        factory = ValidatorFactory()
+        validate = factory(checks=[length(min_chars=100)])
+
+        result = validate("Short")
+        assert not result.passed
+        assert len(result.checks) == 1
+        assert result.checks[0].name == "length"
+
+
+class TestTextValidatorFixture:
+    def test_fixture_returns_factory(self, text_validator: ValidatorFactory) -> None:
+        assert isinstance(text_validator, ValidatorFactory)
+
+    def test_fixture_can_create_validators(
+        self,
+        text_validator: ValidatorFactory,
+    ) -> None:
+        validate = text_validator(checks=[length(min_chars=5, max_chars=50)])
+
+        assert validate("Hello, World!").passed
+        assert not validate("Hi").passed
+
+
+class TestValidationContextFixture:
+    def test_fixture_creates_context(
+        self,
+        validation_context: type,
+    ) -> None:
+        ctx = validation_context(reference="Test reference")
+        assert isinstance(ctx, ValidationContext)
+        assert ctx.reference == "Test reference"
+
+    def test_fixture_accepts_metadata(
+        self,
+        validation_context: type,
+    ) -> None:
+        ctx = validation_context(reference="Test", source="unit_test", version=1)
+        assert ctx.metadata["source"] == "unit_test"
+        assert ctx.metadata["version"] == 1
+
+    def test_fixture_allows_no_reference(
+        self,
+        validation_context: type,
+    ) -> None:
+        ctx = validation_context()
+        assert ctx.reference is None

tests/test_pytest_plugin/test_plugin.py

@@ -0,0 +1,89 @@
+"""Tests for the pytest plugin hooks."""
+
+import pytest
+
+
+@pytest.fixture
+def plugin_pytester(pytester: pytest.Pytester) -> pytest.Pytester:
+    return pytester
+
+
+def test_plugin_registers_marker(plugin_pytester: pytest.Pytester) -> None:
+    plugin_pytester.makepyfile(
+        """
+        import pytest
+
+        @pytest.mark.text_validation
+        def test_example():
+            pass
+        """
+    )
+    # Run with strict markers - this will fail if marker isn't registered
+    result = plugin_pytester.runpytest("--strict-markers")
+    result.assert_outcomes(passed=1)
+
+
+def test_marker_can_be_used(plugin_pytester: pytest.Pytester) -> None:
+    plugin_pytester.makepyfile(
+        """
+        import pytest
+
+        @pytest.mark.text_validation
+        def test_marked():
+            pass
+
+        def test_unmarked():
+            pass
+        """
+    )
+    # Run only marked tests
+    result = plugin_pytester.runpytest("-m", "text_validation")
+    result.assert_outcomes(passed=1)
+
+
+def test_validate_text_is_importable(plugin_pytester: pytest.Pytester) -> None:
+    plugin_pytester.makepyfile(
+        """
+        from veritext.pytest_plugin import validate_text
+
+        def test_import():
+            assert callable(validate_text)
+        """
+    )
+    result = plugin_pytester.runpytest()
+    result.assert_outcomes(passed=1)
+
+
+def test_validate_text_works_in_tests(plugin_pytester: pytest.Pytester) -> None:
+    plugin_pytester.makepyfile(
+        """
+        from veritext.pytest_plugin import validate_text
+
+        def test_validation_passes():
+            validate_text(
+                "The quick brown fox jumps over the lazy dog.",
+                min_length=10,
+                max_length=100,
+            )
+        """
+    )
+    result = plugin_pytester.runpytest()
+    result.assert_outcomes(passed=1)
+
+
+def test_validate_text_failure_in_tests(plugin_pytester: pytest.Pytester) -> None:
+    plugin_pytester.makepyfile(
+        """
+        from veritext.pytest_plugin import validate_text
+
+        def test_validation_fails():
+            validate_text(
+                "Short",
+                min_length=100,
+            )
+        """
+    )
+    result = plugin_pytester.runpytest()
+    result.assert_outcomes(failed=1)
+    # Check that failure message contains useful information
+    result.stdout.fnmatch_lines(["*Text validation failed*"])