Kai Chappell
2827dcdf4e
Cover WordTokeniser (Unicode, empty input, punctuation, multiple scripts) and validation types (immutability, edge cases, failure summary).
21 lines
439 B
Python
21 lines
439 B
Python
"""Shared pytest fixtures for Veritext tests."""
|
|
|
|
import pytest
|
|
|
|
from veritext.core.tokenisation import WordTokeniser
|
|
|
|
|
|
@pytest.fixture
|
|
def word_tokeniser() -> WordTokeniser:
|
|
return WordTokeniser()
|
|
|
|
|
|
@pytest.fixture
|
|
def word_tokeniser_no_lowercase() -> WordTokeniser:
|
|
return WordTokeniser(lowercase=False)
|
|
|
|
|
|
@pytest.fixture
|
|
def word_tokeniser_keep_punctuation() -> WordTokeniser:
|
|
return WordTokeniser(remove_punctuation=False)
|