test(pytest-plugin): add plugin tests

Cover validate_text assertions, fixture factories, marker registration,
and pytest integration using pytester for subprocess testing.

tests/test_pytest_plugin/__init__.py

@@ -0,0 +1 @@
+"""Tests for the Veritext pytest plugin."""

tests/test_pytest_plugin/conftest.py

@@ -0,0 +1,32 @@
+"""Pytest configuration for pytest_plugin tests."""
+
+import pytest
+
+from veritext.pytest_plugin.fixtures import ValidatorFactory
+
+# Enable the pytester fixture for plugin testing
+pytest_plugins = ["pytester"]
+
+# Re-export fixtures from the plugin module for testing
+
+
+@pytest.fixture
+def text_validator() -> ValidatorFactory:
+    """Provide a factory for building validators."""
+    return ValidatorFactory()
+
+
+@pytest.fixture
+def validation_context() -> type:
+    """Provide a factory for creating ValidationContext objects."""
+    from typing import Any
+
+    from veritext.core.types import ValidationContext
+
+    def _create(
+        reference: str | list[str] | None = None,
+        **metadata: Any,
+    ) -> ValidationContext:
+        return ValidationContext(reference=reference, metadata=metadata)
+
+    return _create

tests/test_pytest_plugin/test_assertions.py

@@ -0,0 +1,211 @@
+"""Tests for the validate_text assertion function."""
+
+import pytest
+
+from veritext.pytest_plugin import validate_text
+
+
+class TestValidateTextBasicValidation:
+    """Test basic validation scenarios."""
+
+    def test_passes_with_valid_length(self) -> None:
+        """Test validation passes when length constraints are met."""
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(text, min_length=10, max_length=100)
+
+    def test_fails_when_too_short(self) -> None:
+        """Test validation fails when text is below minimum length."""
+        text = "Short."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, min_length=50)
+        assert "length" in str(exc_info.value).lower()
+
+    def test_fails_when_too_long(self) -> None:
+        """Test validation fails when text exceeds maximum length."""
+        text = "A" * 100
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, max_length=50)
+        assert "length" in str(exc_info.value).lower()
+
+
+class TestValidateTextReadability:
+    """Test readability validation."""
+
+    def test_passes_with_simple_text(self) -> None:
+        """Test validation passes for simple, readable text."""
+        text = "The cat sat on the mat. It was a nice day."
+        validate_text(text, max_reading_grade=10.0)
+
+    def test_fails_with_complex_text(self) -> None:
+        """Test validation fails for overly complex text."""
+        text = (
+            "The implementation of sophisticated metacognitive strategies "
+            "necessitates the comprehensive understanding of epistemological "
+            "frameworks and their corresponding methodological implications."
+        )
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, max_reading_grade=3.0)
+        assert "readability" in str(exc_info.value).lower()
+
+
+class TestValidateTextPatterns:
+    """Test pattern matching validation."""
+
+    def test_passes_when_contains_pattern(self) -> None:
+        """Test validation passes when required pattern is present."""
+        text = "Please contact support@example.com for assistance."
+        validate_text(text, must_contain=["support@example.com"])
+
+    def test_fails_when_missing_required_pattern(self) -> None:
+        """Test validation fails when required pattern is missing."""
+        text = "Please contact us for assistance."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, must_contain=["@example.com"])
+        assert "contains" in str(exc_info.value).lower()
+
+    def test_passes_when_excludes_pattern(self) -> None:
+        """Test validation passes when forbidden pattern is absent."""
+        text = "The report is complete and reviewed."
+        validate_text(text, must_exclude=["TODO", "FIXME"])
+
+    def test_fails_when_contains_forbidden_pattern(self) -> None:
+        """Test validation fails when forbidden pattern is present."""
+        text = "The report is almost done. TODO: add conclusion."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, must_exclude=["TODO"])
+        assert "excludes" in str(exc_info.value).lower()
+
+
+class TestValidateTextComparisonMetrics:
+    """Test comparison-based validation (BLEU, ROUGE)."""
+
+    def test_passes_with_high_bleu_score(self) -> None:
+        """Test validation passes when BLEU score meets threshold."""
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(text, reference=reference, min_bleu=0.9)
+
+    def test_fails_with_low_bleu_score(self) -> None:
+        """Test validation fails when BLEU score is below threshold."""
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "A slow red cat sleeps under the active mouse."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, reference=reference, min_bleu=0.5)
+        assert "bleu" in str(exc_info.value).lower()
+
+    def test_passes_with_high_rouge_score(self) -> None:
+        """Test validation passes when ROUGE score meets threshold."""
+        reference = "Machine learning models require extensive training data."
+        text = "Machine learning models need extensive training data."
+        validate_text(text, reference=reference, min_rouge=0.5)
+
+    def test_fails_with_low_rouge_score(self) -> None:
+        """Test validation fails when ROUGE score is below threshold."""
+        reference = "The algorithm processes input data efficiently."
+        text = "Cats enjoy sleeping in sunny spots."
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, reference=reference, min_rouge=0.5)
+        assert "rouge" in str(exc_info.value).lower()
+
+
+class TestValidateTextErrorHandling:
+    """Test error handling and edge cases."""
+
+    def test_raises_value_error_when_no_criteria(self) -> None:
+        """Test that ValueError is raised when no validation criteria provided."""
+        with pytest.raises(ValueError, match="At least one validation criterion"):
+            validate_text("Some text")
+
+    def test_raises_value_error_when_bleu_without_reference(self) -> None:
+        """Test that ValueError is raised when BLEU requested without reference."""
+        with pytest.raises(ValueError, match="Reference text required"):
+            validate_text("Some text", min_bleu=0.5)
+
+    def test_raises_value_error_when_rouge_without_reference(self) -> None:
+        """Test that ValueError is raised when ROUGE requested without reference."""
+        with pytest.raises(ValueError, match="Reference text required"):
+            validate_text("Some text", min_rouge=0.5)
+
+    def test_raises_value_error_when_semantic_without_reference(self) -> None:
+        """Test that ValueError is raised for semantic without reference."""
+        with pytest.raises(ValueError, match="Reference text required"):
+            validate_text("Some text", min_semantic=0.5)
+
+
+class TestValidateTextMultipleCriteria:
+    """Test validation with multiple criteria combined."""
+
+    def test_passes_all_criteria(self) -> None:
+        """Test validation passes when all criteria are met."""
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(
+            text,
+            reference=reference,
+            min_bleu=0.9,
+            min_length=10,
+            max_length=100,
+        )
+
+    def test_fails_when_one_criterion_fails(self) -> None:
+        """Test validation fails when any criterion fails."""
+        reference = "The quick brown fox jumps over the lazy dog."
+        text = "The quick brown fox jumps over the lazy dog."
+        with pytest.raises(AssertionError):
+            validate_text(
+                text,
+                reference=reference,
+                min_bleu=0.9,
+                max_length=10,  # This will fail
+            )
+
+
+class TestValidateTextFailureMessage:
+    """Test failure message formatting."""
+
+    def test_failure_message_includes_text_preview(self) -> None:
+        """Test that failure message includes preview of the text."""
+        text = "Short text"
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, min_length=100)
+        assert "Short text" in str(exc_info.value)
+
+    def test_failure_message_truncates_long_text(self) -> None:
+        """Test that long text is truncated in failure message."""
+        text = "A" * 200
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, max_length=50)
+        message = str(exc_info.value)
+        assert "..." in message
+        assert "A" * 200 not in message
+
+    def test_failure_message_includes_check_details(self) -> None:
+        """Test that failure message includes check name and details."""
+        text = "Short"
+        with pytest.raises(AssertionError) as exc_info:
+            validate_text(text, min_length=100)
+        message = str(exc_info.value)
+        assert "Failed checks:" in message
+        assert "length" in message.lower()
+
+
+class TestValidateTextListReference:
+    """Test validation with list of reference texts."""
+
+    def test_bleu_with_multiple_references(self) -> None:
+        """Test BLEU validation accepts multiple reference texts."""
+        references = [
+            "The quick brown fox jumps over the lazy dog.",
+            "A fast brown fox leaps over a sleepy dog.",
+        ]
+        text = "The quick brown fox jumps over the lazy dog."
+        validate_text(text, reference=references, min_bleu=0.9)
+
+    def test_rouge_with_multiple_references(self) -> None:
+        """Test ROUGE validation accepts multiple reference texts."""
+        references = [
+            "Machine learning requires data.",
+            "ML models need training data.",
+        ]
+        text = "Machine learning models require training data."
+        validate_text(text, reference=references, min_rouge=0.3)

tests/test_pytest_plugin/test_fixtures.py

@@ -0,0 +1,88 @@
+"""Tests for the pytest plugin fixtures."""
+
+from veritext.core.types import ValidationContext
+from veritext.pytest_plugin.fixtures import ValidatorFactory
+from veritext.validators import bleu, length
+
+
+class TestValidatorFactory:
+    """Test the ValidatorFactory class."""
+
+    def test_creates_validator_from_checks(self) -> None:
+        """Test that factory creates a callable validator."""
+        factory = ValidatorFactory()
+        validate = factory(checks=[length(min_chars=5)])
+
+        result = validate("Hello, World!")
+        assert result.passed
+
+    def test_validator_uses_provided_reference(self) -> None:
+        """Test that factory passes reference to context."""
+        factory = ValidatorFactory()
+        reference = "The quick brown fox."
+        validate = factory(
+            checks=[bleu(min_score=0.5)],
+            reference=reference,
+        )
+
+        # Exact match should pass
+        result = validate("The quick brown fox.")
+        assert result.passed
+
+    def test_validator_returns_validation_result(self) -> None:
+        """Test that validator returns a ValidationResult."""
+        factory = ValidatorFactory()
+        validate = factory(checks=[length(min_chars=100)])
+
+        result = validate("Short")
+        assert not result.passed
+        assert len(result.checks) == 1
+        assert result.checks[0].name == "length"
+
+
+class TestTextValidatorFixture:
+    """Test the text_validator fixture."""
+
+    def test_fixture_returns_factory(self, text_validator: ValidatorFactory) -> None:
+        """Test that fixture provides a ValidatorFactory."""
+        assert isinstance(text_validator, ValidatorFactory)
+
+    def test_fixture_can_create_validators(
+        self,
+        text_validator: ValidatorFactory,
+    ) -> None:
+        """Test that fixture can be used to create validators."""
+        validate = text_validator(checks=[length(min_chars=5, max_chars=50)])
+
+        assert validate("Hello, World!").passed
+        assert not validate("Hi").passed
+
+
+class TestValidationContextFixture:
+    """Test the validation_context fixture."""
+
+    def test_fixture_creates_context(
+        self,
+        validation_context: type,
+    ) -> None:
+        """Test that fixture creates ValidationContext."""
+        ctx = validation_context(reference="Test reference")
+        assert isinstance(ctx, ValidationContext)
+        assert ctx.reference == "Test reference"
+
+    def test_fixture_accepts_metadata(
+        self,
+        validation_context: type,
+    ) -> None:
+        """Test that fixture passes metadata to context."""
+        ctx = validation_context(reference="Test", source="unit_test", version=1)
+        assert ctx.metadata["source"] == "unit_test"
+        assert ctx.metadata["version"] == 1
+
+    def test_fixture_allows_no_reference(
+        self,
+        validation_context: type,
+    ) -> None:
+        """Test that fixture allows creating context without reference."""
+        ctx = validation_context()
+        assert ctx.reference is None

tests/test_pytest_plugin/test_plugin.py

@@ -0,0 +1,100 @@
+"""Tests for the pytest plugin hooks."""
+
+import pytest
+
+
+@pytest.fixture
+def plugin_pytester(pytester: pytest.Pytester) -> pytest.Pytester:
+    """Configure pytester to use the veritext plugin."""
+    pytester.makeconftest(
+        """
+        pytest_plugins = ['veritext.pytest_plugin']
+        """
+    )
+    return pytester
+
+
+def test_plugin_registers_marker(plugin_pytester: pytest.Pytester) -> None:
+    """Test that the text_validation marker is registered."""
+    plugin_pytester.makepyfile(
+        """
+        import pytest
+
+        @pytest.mark.text_validation
+        def test_example():
+            pass
+        """
+    )
+    # Run with strict markers - this will fail if marker isn't registered
+    result = plugin_pytester.runpytest("--strict-markers")
+    result.assert_outcomes(passed=1)
+
+
+def test_marker_can_be_used(plugin_pytester: pytest.Pytester) -> None:
+    """Test that the text_validation marker can filter tests."""
+    plugin_pytester.makepyfile(
+        """
+        import pytest
+
+        @pytest.mark.text_validation
+        def test_marked():
+            pass
+
+        def test_unmarked():
+            pass
+        """
+    )
+    # Run only marked tests
+    result = plugin_pytester.runpytest("-m", "text_validation")
+    result.assert_outcomes(passed=1)
+
+
+def test_validate_text_is_importable(plugin_pytester: pytest.Pytester) -> None:
+    """Test that validate_text can be imported from the plugin."""
+    plugin_pytester.makepyfile(
+        """
+        from veritext.pytest_plugin import validate_text
+
+        def test_import():
+            assert callable(validate_text)
+        """
+    )
+    result = plugin_pytester.runpytest()
+    result.assert_outcomes(passed=1)
+
+
+def test_validate_text_works_in_tests(plugin_pytester: pytest.Pytester) -> None:
+    """Test that validate_text can be used in test functions."""
+    plugin_pytester.makepyfile(
+        """
+        from veritext.pytest_plugin import validate_text
+
+        def test_validation_passes():
+            validate_text(
+                "The quick brown fox jumps over the lazy dog.",
+                min_length=10,
+                max_length=100,
+            )
+        """
+    )
+    result = plugin_pytester.runpytest()
+    result.assert_outcomes(passed=1)
+
+
+def test_validate_text_failure_in_tests(plugin_pytester: pytest.Pytester) -> None:
+    """Test that validate_text failures are reported properly."""
+    plugin_pytester.makepyfile(
+        """
+        from veritext.pytest_plugin import validate_text
+
+        def test_validation_fails():
+            validate_text(
+                "Short",
+                min_length=100,
+            )
+        """
+    )
+    result = plugin_pytester.runpytest()
+    result.assert_outcomes(failed=1)
+    # Check that failure message contains useful information
+    result.stdout.fnmatch_lines(["*Text validation failed*"])