diff --git a/tests/test_pytest_plugin/__init__.py b/tests/test_pytest_plugin/__init__.py new file mode 100644 index 0000000..3553062 --- /dev/null +++ b/tests/test_pytest_plugin/__init__.py @@ -0,0 +1 @@ +"""Tests for the Veritext pytest plugin.""" diff --git a/tests/test_pytest_plugin/conftest.py b/tests/test_pytest_plugin/conftest.py new file mode 100644 index 0000000..c8cf41a --- /dev/null +++ b/tests/test_pytest_plugin/conftest.py @@ -0,0 +1,32 @@ +"""Pytest configuration for pytest_plugin tests.""" + +import pytest + +from veritext.pytest_plugin.fixtures import ValidatorFactory + +# Enable the pytester fixture for plugin testing +pytest_plugins = ["pytester"] + +# Re-export fixtures from the plugin module for testing + + +@pytest.fixture +def text_validator() -> ValidatorFactory: + """Provide a factory for building validators.""" + return ValidatorFactory() + + +@pytest.fixture +def validation_context() -> type: + """Provide a factory for creating ValidationContext objects.""" + from typing import Any + + from veritext.core.types import ValidationContext + + def _create( + reference: str | list[str] | None = None, + **metadata: Any, + ) -> ValidationContext: + return ValidationContext(reference=reference, metadata=metadata) + + return _create diff --git a/tests/test_pytest_plugin/test_assertions.py b/tests/test_pytest_plugin/test_assertions.py new file mode 100644 index 0000000..7fba32d --- /dev/null +++ b/tests/test_pytest_plugin/test_assertions.py @@ -0,0 +1,211 @@ +"""Tests for the validate_text assertion function.""" + +import pytest + +from veritext.pytest_plugin import validate_text + + +class TestValidateTextBasicValidation: + """Test basic validation scenarios.""" + + def test_passes_with_valid_length(self) -> None: + """Test validation passes when length constraints are met.""" + text = "The quick brown fox jumps over the lazy dog." + validate_text(text, min_length=10, max_length=100) + + def test_fails_when_too_short(self) -> None: + """Test validation fails when text is below minimum length.""" + text = "Short." + with pytest.raises(AssertionError) as exc_info: + validate_text(text, min_length=50) + assert "length" in str(exc_info.value).lower() + + def test_fails_when_too_long(self) -> None: + """Test validation fails when text exceeds maximum length.""" + text = "A" * 100 + with pytest.raises(AssertionError) as exc_info: + validate_text(text, max_length=50) + assert "length" in str(exc_info.value).lower() + + +class TestValidateTextReadability: + """Test readability validation.""" + + def test_passes_with_simple_text(self) -> None: + """Test validation passes for simple, readable text.""" + text = "The cat sat on the mat. It was a nice day." + validate_text(text, max_reading_grade=10.0) + + def test_fails_with_complex_text(self) -> None: + """Test validation fails for overly complex text.""" + text = ( + "The implementation of sophisticated metacognitive strategies " + "necessitates the comprehensive understanding of epistemological " + "frameworks and their corresponding methodological implications." + ) + with pytest.raises(AssertionError) as exc_info: + validate_text(text, max_reading_grade=3.0) + assert "readability" in str(exc_info.value).lower() + + +class TestValidateTextPatterns: + """Test pattern matching validation.""" + + def test_passes_when_contains_pattern(self) -> None: + """Test validation passes when required pattern is present.""" + text = "Please contact support@example.com for assistance." + validate_text(text, must_contain=["support@example.com"]) + + def test_fails_when_missing_required_pattern(self) -> None: + """Test validation fails when required pattern is missing.""" + text = "Please contact us for assistance." + with pytest.raises(AssertionError) as exc_info: + validate_text(text, must_contain=["@example.com"]) + assert "contains" in str(exc_info.value).lower() + + def test_passes_when_excludes_pattern(self) -> None: + """Test validation passes when forbidden pattern is absent.""" + text = "The report is complete and reviewed." + validate_text(text, must_exclude=["TODO", "FIXME"]) + + def test_fails_when_contains_forbidden_pattern(self) -> None: + """Test validation fails when forbidden pattern is present.""" + text = "The report is almost done. TODO: add conclusion." + with pytest.raises(AssertionError) as exc_info: + validate_text(text, must_exclude=["TODO"]) + assert "excludes" in str(exc_info.value).lower() + + +class TestValidateTextComparisonMetrics: + """Test comparison-based validation (BLEU, ROUGE).""" + + def test_passes_with_high_bleu_score(self) -> None: + """Test validation passes when BLEU score meets threshold.""" + reference = "The quick brown fox jumps over the lazy dog." + text = "The quick brown fox jumps over the lazy dog." + validate_text(text, reference=reference, min_bleu=0.9) + + def test_fails_with_low_bleu_score(self) -> None: + """Test validation fails when BLEU score is below threshold.""" + reference = "The quick brown fox jumps over the lazy dog." + text = "A slow red cat sleeps under the active mouse." + with pytest.raises(AssertionError) as exc_info: + validate_text(text, reference=reference, min_bleu=0.5) + assert "bleu" in str(exc_info.value).lower() + + def test_passes_with_high_rouge_score(self) -> None: + """Test validation passes when ROUGE score meets threshold.""" + reference = "Machine learning models require extensive training data." + text = "Machine learning models need extensive training data." + validate_text(text, reference=reference, min_rouge=0.5) + + def test_fails_with_low_rouge_score(self) -> None: + """Test validation fails when ROUGE score is below threshold.""" + reference = "The algorithm processes input data efficiently." + text = "Cats enjoy sleeping in sunny spots." + with pytest.raises(AssertionError) as exc_info: + validate_text(text, reference=reference, min_rouge=0.5) + assert "rouge" in str(exc_info.value).lower() + + +class TestValidateTextErrorHandling: + """Test error handling and edge cases.""" + + def test_raises_value_error_when_no_criteria(self) -> None: + """Test that ValueError is raised when no validation criteria provided.""" + with pytest.raises(ValueError, match="At least one validation criterion"): + validate_text("Some text") + + def test_raises_value_error_when_bleu_without_reference(self) -> None: + """Test that ValueError is raised when BLEU requested without reference.""" + with pytest.raises(ValueError, match="Reference text required"): + validate_text("Some text", min_bleu=0.5) + + def test_raises_value_error_when_rouge_without_reference(self) -> None: + """Test that ValueError is raised when ROUGE requested without reference.""" + with pytest.raises(ValueError, match="Reference text required"): + validate_text("Some text", min_rouge=0.5) + + def test_raises_value_error_when_semantic_without_reference(self) -> None: + """Test that ValueError is raised for semantic without reference.""" + with pytest.raises(ValueError, match="Reference text required"): + validate_text("Some text", min_semantic=0.5) + + +class TestValidateTextMultipleCriteria: + """Test validation with multiple criteria combined.""" + + def test_passes_all_criteria(self) -> None: + """Test validation passes when all criteria are met.""" + reference = "The quick brown fox jumps over the lazy dog." + text = "The quick brown fox jumps over the lazy dog." + validate_text( + text, + reference=reference, + min_bleu=0.9, + min_length=10, + max_length=100, + ) + + def test_fails_when_one_criterion_fails(self) -> None: + """Test validation fails when any criterion fails.""" + reference = "The quick brown fox jumps over the lazy dog." + text = "The quick brown fox jumps over the lazy dog." + with pytest.raises(AssertionError): + validate_text( + text, + reference=reference, + min_bleu=0.9, + max_length=10, # This will fail + ) + + +class TestValidateTextFailureMessage: + """Test failure message formatting.""" + + def test_failure_message_includes_text_preview(self) -> None: + """Test that failure message includes preview of the text.""" + text = "Short text" + with pytest.raises(AssertionError) as exc_info: + validate_text(text, min_length=100) + assert "Short text" in str(exc_info.value) + + def test_failure_message_truncates_long_text(self) -> None: + """Test that long text is truncated in failure message.""" + text = "A" * 200 + with pytest.raises(AssertionError) as exc_info: + validate_text(text, max_length=50) + message = str(exc_info.value) + assert "..." in message + assert "A" * 200 not in message + + def test_failure_message_includes_check_details(self) -> None: + """Test that failure message includes check name and details.""" + text = "Short" + with pytest.raises(AssertionError) as exc_info: + validate_text(text, min_length=100) + message = str(exc_info.value) + assert "Failed checks:" in message + assert "length" in message.lower() + + +class TestValidateTextListReference: + """Test validation with list of reference texts.""" + + def test_bleu_with_multiple_references(self) -> None: + """Test BLEU validation accepts multiple reference texts.""" + references = [ + "The quick brown fox jumps over the lazy dog.", + "A fast brown fox leaps over a sleepy dog.", + ] + text = "The quick brown fox jumps over the lazy dog." + validate_text(text, reference=references, min_bleu=0.9) + + def test_rouge_with_multiple_references(self) -> None: + """Test ROUGE validation accepts multiple reference texts.""" + references = [ + "Machine learning requires data.", + "ML models need training data.", + ] + text = "Machine learning models require training data." + validate_text(text, reference=references, min_rouge=0.3) diff --git a/tests/test_pytest_plugin/test_fixtures.py b/tests/test_pytest_plugin/test_fixtures.py new file mode 100644 index 0000000..a65e2b4 --- /dev/null +++ b/tests/test_pytest_plugin/test_fixtures.py @@ -0,0 +1,88 @@ +"""Tests for the pytest plugin fixtures.""" + +from veritext.core.types import ValidationContext +from veritext.pytest_plugin.fixtures import ValidatorFactory +from veritext.validators import bleu, length + + +class TestValidatorFactory: + """Test the ValidatorFactory class.""" + + def test_creates_validator_from_checks(self) -> None: + """Test that factory creates a callable validator.""" + factory = ValidatorFactory() + validate = factory(checks=[length(min_chars=5)]) + + result = validate("Hello, World!") + assert result.passed + + def test_validator_uses_provided_reference(self) -> None: + """Test that factory passes reference to context.""" + factory = ValidatorFactory() + reference = "The quick brown fox." + validate = factory( + checks=[bleu(min_score=0.5)], + reference=reference, + ) + + # Exact match should pass + result = validate("The quick brown fox.") + assert result.passed + + def test_validator_returns_validation_result(self) -> None: + """Test that validator returns a ValidationResult.""" + factory = ValidatorFactory() + validate = factory(checks=[length(min_chars=100)]) + + result = validate("Short") + assert not result.passed + assert len(result.checks) == 1 + assert result.checks[0].name == "length" + + +class TestTextValidatorFixture: + """Test the text_validator fixture.""" + + def test_fixture_returns_factory(self, text_validator: ValidatorFactory) -> None: + """Test that fixture provides a ValidatorFactory.""" + assert isinstance(text_validator, ValidatorFactory) + + def test_fixture_can_create_validators( + self, + text_validator: ValidatorFactory, + ) -> None: + """Test that fixture can be used to create validators.""" + validate = text_validator(checks=[length(min_chars=5, max_chars=50)]) + + assert validate("Hello, World!").passed + assert not validate("Hi").passed + + +class TestValidationContextFixture: + """Test the validation_context fixture.""" + + def test_fixture_creates_context( + self, + validation_context: type, + ) -> None: + """Test that fixture creates ValidationContext.""" + ctx = validation_context(reference="Test reference") + assert isinstance(ctx, ValidationContext) + assert ctx.reference == "Test reference" + + def test_fixture_accepts_metadata( + self, + validation_context: type, + ) -> None: + """Test that fixture passes metadata to context.""" + ctx = validation_context(reference="Test", source="unit_test", version=1) + assert ctx.metadata["source"] == "unit_test" + assert ctx.metadata["version"] == 1 + + def test_fixture_allows_no_reference( + self, + validation_context: type, + ) -> None: + """Test that fixture allows creating context without reference.""" + ctx = validation_context() + assert ctx.reference is None diff --git a/tests/test_pytest_plugin/test_plugin.py b/tests/test_pytest_plugin/test_plugin.py new file mode 100644 index 0000000..6efa2c5 --- /dev/null +++ b/tests/test_pytest_plugin/test_plugin.py @@ -0,0 +1,100 @@ +"""Tests for the pytest plugin hooks.""" + +import pytest + + +@pytest.fixture +def plugin_pytester(pytester: pytest.Pytester) -> pytest.Pytester: + """Configure pytester to use the veritext plugin.""" + pytester.makeconftest( + """ + pytest_plugins = ['veritext.pytest_plugin'] + """ + ) + return pytester + + +def test_plugin_registers_marker(plugin_pytester: pytest.Pytester) -> None: + """Test that the text_validation marker is registered.""" + plugin_pytester.makepyfile( + """ + import pytest + + @pytest.mark.text_validation + def test_example(): + pass + """ + ) + # Run with strict markers - this will fail if marker isn't registered + result = plugin_pytester.runpytest("--strict-markers") + result.assert_outcomes(passed=1) + + +def test_marker_can_be_used(plugin_pytester: pytest.Pytester) -> None: + """Test that the text_validation marker can filter tests.""" + plugin_pytester.makepyfile( + """ + import pytest + + @pytest.mark.text_validation + def test_marked(): + pass + + def test_unmarked(): + pass + """ + ) + # Run only marked tests + result = plugin_pytester.runpytest("-m", "text_validation") + result.assert_outcomes(passed=1) + + +def test_validate_text_is_importable(plugin_pytester: pytest.Pytester) -> None: + """Test that validate_text can be imported from the plugin.""" + plugin_pytester.makepyfile( + """ + from veritext.pytest_plugin import validate_text + + def test_import(): + assert callable(validate_text) + """ + ) + result = plugin_pytester.runpytest() + result.assert_outcomes(passed=1) + + +def test_validate_text_works_in_tests(plugin_pytester: pytest.Pytester) -> None: + """Test that validate_text can be used in test functions.""" + plugin_pytester.makepyfile( + """ + from veritext.pytest_plugin import validate_text + + def test_validation_passes(): + validate_text( + "The quick brown fox jumps over the lazy dog.", + min_length=10, + max_length=100, + ) + """ + ) + result = plugin_pytester.runpytest() + result.assert_outcomes(passed=1) + + +def test_validate_text_failure_in_tests(plugin_pytester: pytest.Pytester) -> None: + """Test that validate_text failures are reported properly.""" + plugin_pytester.makepyfile( + """ + from veritext.pytest_plugin import validate_text + + def test_validation_fails(): + validate_text( + "Short", + min_length=100, + ) + """ + ) + result = plugin_pytester.runpytest() + result.assert_outcomes(failed=1) + # Check that failure message contains useful information + result.stdout.fnmatch_lines(["*Text validation failed*"])