Kai Chappell
1e3618e637
Cover WordTokeniser (Unicode, empty input, punctuation, multiple scripts) and validation types (immutability, edge cases, failure summary).
24 lines
598 B
Python
24 lines
598 B
Python
"""Shared pytest fixtures for Veritext tests."""
|
|
|
|
import pytest
|
|
|
|
from veritext.core.tokenisation import WordTokeniser
|
|
|
|
|
|
@pytest.fixture
|
|
def word_tokeniser() -> WordTokeniser:
|
|
"""Provide a default word tokeniser."""
|
|
return WordTokeniser()
|
|
|
|
|
|
@pytest.fixture
|
|
def word_tokeniser_no_lowercase() -> WordTokeniser:
|
|
"""Provide a word tokeniser without lowercasing."""
|
|
return WordTokeniser(lowercase=False)
|
|
|
|
|
|
@pytest.fixture
|
|
def word_tokeniser_keep_punctuation() -> WordTokeniser:
|
|
"""Provide a word tokeniser that keeps punctuation."""
|
|
return WordTokeniser(remove_punctuation=False)
|