From 4d103cbe52dbdc053352c317898a1028b975d35c Mon Sep 17 00:00:00 2001 From: Kai Chappell Date: Sun, 6 Apr 2025 15:50:41 +0000 Subject: [PATCH] test pytest plugin Cover validate_text assertions, fixture factories, marker registration, and pytest integration using pytester for subprocess testing. --- tests/test_pytest_plugin/__init__.py | 1 + tests/test_pytest_plugin/conftest.py | 30 ++++ tests/test_pytest_plugin/test_assertions.py | 171 ++++++++++++++++++++ tests/test_pytest_plugin/test_fixtures.py | 74 +++++++++ tests/test_pytest_plugin/test_plugin.py | 89 ++++++++++ 5 files changed, 365 insertions(+) create mode 100644 tests/test_pytest_plugin/__init__.py create mode 100644 tests/test_pytest_plugin/conftest.py create mode 100644 tests/test_pytest_plugin/test_assertions.py create mode 100644 tests/test_pytest_plugin/test_fixtures.py create mode 100644 tests/test_pytest_plugin/test_plugin.py diff --git a/tests/test_pytest_plugin/__init__.py b/tests/test_pytest_plugin/__init__.py new file mode 100644 index 0000000..3553062 --- /dev/null +++ b/tests/test_pytest_plugin/__init__.py @@ -0,0 +1 @@ +"""Tests for the Veritext pytest plugin.""" diff --git a/tests/test_pytest_plugin/conftest.py b/tests/test_pytest_plugin/conftest.py new file mode 100644 index 0000000..88b8171 --- /dev/null +++ b/tests/test_pytest_plugin/conftest.py @@ -0,0 +1,30 @@ +"""Pytest configuration for pytest_plugin tests.""" + +import pytest + +from veritext.pytest_plugin.fixtures import ValidatorFactory + +# Enable the pytester fixture for plugin testing +pytest_plugins = ["pytester"] + +# Re-export fixtures from the plugin module for testing + + +@pytest.fixture +def text_validator() -> ValidatorFactory: + return ValidatorFactory() + + +@pytest.fixture +def validation_context() -> type: + from typing import Any + + from veritext.core.types import ValidationContext + + def _create( + reference: str | list[str] | None = None, + **metadata: Any, + ) -> ValidationContext: + return ValidationContext(reference=reference, metadata=metadata) + + return _create diff --git a/tests/test_pytest_plugin/test_assertions.py b/tests/test_pytest_plugin/test_assertions.py new file mode 100644 index 0000000..3ff6210 --- /dev/null +++ b/tests/test_pytest_plugin/test_assertions.py @@ -0,0 +1,171 @@ +"""Tests for the validate_text assertion function.""" + +import pytest + +from veritext.pytest_plugin import validate_text + + +class TestValidateTextBasicValidation: + def test_valid_length(self) -> None: + text = "The quick brown fox jumps over the lazy dog." + validate_text(text, min_length=10, max_length=100) + + def test_too_short(self) -> None: + text = "Short." + with pytest.raises(AssertionError) as exc_info: + validate_text(text, min_length=50) + assert "length" in str(exc_info.value).lower() + + def test_too_long(self) -> None: + text = "A" * 100 + with pytest.raises(AssertionError) as exc_info: + validate_text(text, max_length=50) + assert "length" in str(exc_info.value).lower() + + +class TestValidateTextReadability: + def test_simple_text_passes(self) -> None: + text = "The cat sat on the mat. It was a nice day." + validate_text(text, max_reading_grade=10.0) + + def test_complex_text_fails(self) -> None: + text = ( + "The implementation of sophisticated metacognitive strategies " + "necessitates the thorough understanding of epistemological " + "frameworks and their corresponding methodological implications." + ) + with pytest.raises(AssertionError) as exc_info: + validate_text(text, max_reading_grade=3.0) + assert "readability" in str(exc_info.value).lower() + + +class TestValidateTextPatterns: + def test_contains_pattern(self) -> None: + text = "Please contact support@example.com for assistance." + validate_text(text, must_contain=["support@example.com"]) + + def test_missing_pattern(self) -> None: + text = "Please contact us for assistance." + with pytest.raises(AssertionError) as exc_info: + validate_text(text, must_contain=["@example.com"]) + assert "contains" in str(exc_info.value).lower() + + def test_excludes_pattern(self) -> None: + text = "The report is complete and reviewed." + validate_text(text, must_exclude=["TODO", "FIXME"]) + + def test_forbidden_pattern(self) -> None: + text = "The report is almost done. TODO: add conclusion." + with pytest.raises(AssertionError) as exc_info: + validate_text(text, must_exclude=["TODO"]) + assert "excludes" in str(exc_info.value).lower() + + +class TestValidateTextComparisonMetrics: + def test_high_bleu_passes(self) -> None: + reference = "The quick brown fox jumps over the lazy dog." + text = "The quick brown fox jumps over the lazy dog." + validate_text(text, reference=reference, min_bleu=0.9) + + def test_low_bleu_fails(self) -> None: + reference = "The quick brown fox jumps over the lazy dog." + text = "A slow red cat sleeps under the active mouse." + with pytest.raises(AssertionError) as exc_info: + validate_text(text, reference=reference, min_bleu=0.5) + assert "bleu" in str(exc_info.value).lower() + + def test_high_rouge_passes(self) -> None: + reference = "Machine learning models require extensive training data." + text = "Machine learning models need extensive training data." + validate_text(text, reference=reference, min_rouge=0.5) + + def test_low_rouge_fails(self) -> None: + reference = "The algorithm processes input data efficiently." + text = "Cats enjoy sleeping in sunny spots." + with pytest.raises(AssertionError) as exc_info: + validate_text(text, reference=reference, min_rouge=0.5) + assert "rouge" in str(exc_info.value).lower() + + +class TestValidateTextErrorHandling: + def test_requires_criteria(self) -> None: + with pytest.raises(ValueError, match="At least one validation criterion"): + validate_text("Some text") + + def test_bleu_requires_reference(self) -> None: + with pytest.raises(ValueError, match="Reference text required"): + validate_text("Some text", min_bleu=0.5) + + def test_rouge_requires_reference(self) -> None: + with pytest.raises(ValueError, match="Reference text required"): + validate_text("Some text", min_rouge=0.5) + + def test_semantic_requires_reference(self) -> None: + with pytest.raises(ValueError, match="Reference text required"): + validate_text("Some text", min_semantic=0.5) + + +class TestValidateTextMultipleCriteria: + def test_all_criteria_pass(self) -> None: + reference = "The quick brown fox jumps over the lazy dog." + text = "The quick brown fox jumps over the lazy dog." + validate_text( + text, + reference=reference, + min_bleu=0.9, + min_length=10, + max_length=100, + ) + + def test_one_failure_fails_all(self) -> None: + reference = "The quick brown fox jumps over the lazy dog." + text = "The quick brown fox jumps over the lazy dog." + with pytest.raises(AssertionError): + validate_text( + text, + reference=reference, + min_bleu=0.9, + max_length=10, # This will fail + ) + + +class TestValidateTextFailureMessage: + def test_includes_text_preview(self) -> None: + text = "Short text" + with pytest.raises(AssertionError) as exc_info: + validate_text(text, min_length=100) + assert "Short text" in str(exc_info.value) + + def test_truncates_long_text(self) -> None: + text = "A" * 200 + with pytest.raises(AssertionError) as exc_info: + validate_text(text, max_length=50) + message = str(exc_info.value) + assert "..." in message + assert "A" * 200 not in message + + def test_includes_check_details(self) -> None: + text = "Short" + with pytest.raises(AssertionError) as exc_info: + validate_text(text, min_length=100) + message = str(exc_info.value) + assert "Failed checks:" in message + assert "length" in message.lower() + + +class TestValidateTextListReference: + def test_bleu_multi_reference(self) -> None: + references = [ + "The quick brown fox jumps over the lazy dog.", + "A fast brown fox leaps over a sleepy dog.", + ] + text = "The quick brown fox jumps over the lazy dog." + validate_text(text, reference=references, min_bleu=0.9) + + def test_rouge_multi_reference(self) -> None: + references = [ + "Machine learning requires data.", + "ML models need training data.", + ] + text = "Machine learning models require training data." + validate_text(text, reference=references, min_rouge=0.3) diff --git a/tests/test_pytest_plugin/test_fixtures.py b/tests/test_pytest_plugin/test_fixtures.py new file mode 100644 index 0000000..fd66984 --- /dev/null +++ b/tests/test_pytest_plugin/test_fixtures.py @@ -0,0 +1,74 @@ +"""Tests for the pytest plugin fixtures.""" + +from veritext.core.types import ValidationContext +from veritext.pytest_plugin.fixtures import ValidatorFactory +from veritext.validators import bleu, length + + +class TestValidatorFactory: + def test_creates_validator_from_checks(self) -> None: + factory = ValidatorFactory() + validate = factory(checks=[length(min_chars=5)]) + + result = validate("Hello, World!") + assert result.passed + + def test_validator_uses_provided_reference(self) -> None: + factory = ValidatorFactory() + reference = "The quick brown fox." + validate = factory( + checks=[bleu(min_score=0.5)], + reference=reference, + ) + + # Exact match should pass + result = validate("The quick brown fox.") + assert result.passed + + def test_validator_returns_validation_result(self) -> None: + factory = ValidatorFactory() + validate = factory(checks=[length(min_chars=100)]) + + result = validate("Short") + assert not result.passed + assert len(result.checks) == 1 + assert result.checks[0].name == "length" + + +class TestTextValidatorFixture: + def test_fixture_returns_factory(self, text_validator: ValidatorFactory) -> None: + assert isinstance(text_validator, ValidatorFactory) + + def test_fixture_can_create_validators( + self, + text_validator: ValidatorFactory, + ) -> None: + validate = text_validator(checks=[length(min_chars=5, max_chars=50)]) + + assert validate("Hello, World!").passed + assert not validate("Hi").passed + + +class TestValidationContextFixture: + def test_fixture_creates_context( + self, + validation_context: type, + ) -> None: + ctx = validation_context(reference="Test reference") + assert isinstance(ctx, ValidationContext) + assert ctx.reference == "Test reference" + + def test_fixture_accepts_metadata( + self, + validation_context: type, + ) -> None: + ctx = validation_context(reference="Test", source="unit_test", version=1) + assert ctx.metadata["source"] == "unit_test" + assert ctx.metadata["version"] == 1 + + def test_fixture_allows_no_reference( + self, + validation_context: type, + ) -> None: + ctx = validation_context() + assert ctx.reference is None diff --git a/tests/test_pytest_plugin/test_plugin.py b/tests/test_pytest_plugin/test_plugin.py new file mode 100644 index 0000000..9ddf940 --- /dev/null +++ b/tests/test_pytest_plugin/test_plugin.py @@ -0,0 +1,89 @@ +"""Tests for the pytest plugin hooks.""" + +import pytest + + +@pytest.fixture +def plugin_pytester(pytester: pytest.Pytester) -> pytest.Pytester: + return pytester + + +def test_plugin_registers_marker(plugin_pytester: pytest.Pytester) -> None: + plugin_pytester.makepyfile( + """ + import pytest + + @pytest.mark.text_validation + def test_example(): + pass + """ + ) + # Run with strict markers - this will fail if marker isn't registered + result = plugin_pytester.runpytest("--strict-markers") + result.assert_outcomes(passed=1) + + +def test_marker_can_be_used(plugin_pytester: pytest.Pytester) -> None: + plugin_pytester.makepyfile( + """ + import pytest + + @pytest.mark.text_validation + def test_marked(): + pass + + def test_unmarked(): + pass + """ + ) + # Run only marked tests + result = plugin_pytester.runpytest("-m", "text_validation") + result.assert_outcomes(passed=1) + + +def test_validate_text_is_importable(plugin_pytester: pytest.Pytester) -> None: + plugin_pytester.makepyfile( + """ + from veritext.pytest_plugin import validate_text + + def test_import(): + assert callable(validate_text) + """ + ) + result = plugin_pytester.runpytest() + result.assert_outcomes(passed=1) + + +def test_validate_text_works_in_tests(plugin_pytester: pytest.Pytester) -> None: + plugin_pytester.makepyfile( + """ + from veritext.pytest_plugin import validate_text + + def test_validation_passes(): + validate_text( + "The quick brown fox jumps over the lazy dog.", + min_length=10, + max_length=100, + ) + """ + ) + result = plugin_pytester.runpytest() + result.assert_outcomes(passed=1) + + +def test_validate_text_failure_in_tests(plugin_pytester: pytest.Pytester) -> None: + plugin_pytester.makepyfile( + """ + from veritext.pytest_plugin import validate_text + + def test_validation_fails(): + validate_text( + "Short", + min_length=100, + ) + """ + ) + result = plugin_pytester.runpytest() + result.assert_outcomes(failed=1) + # Check that failure message contains useful information + result.stdout.fnmatch_lines(["*Text validation failed*"])