tests for tokeniser and types

Cover WordTokeniser (Unicode, empty input, punctuation, multiple scripts)
and validation types (immutability, edge cases, failure summary).

tests/test_core/test_tokenisation.py

@@ -0,0 +1,103 @@
+"""Tests for the tokenisation module."""
+
+from typing import TYPE_CHECKING
+
+from veritext.core.tokenisation import WordTokeniser
+
+if TYPE_CHECKING:
+    from veritext.core.tokenisation import Tokeniser
+
+
+class TestWordTokeniser:
+    def test_basic_tokenisation(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("The cat sat on the mat")
+        assert tokens == ["the", "cat", "sat", "on", "the", "mat"]
+
+    def test_lowercasing(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("Hello WORLD")
+        assert tokens == ["hello", "world"]
+
+    def test_no_lowercasing(self, word_tokeniser_no_lowercase: WordTokeniser) -> None:
+        tokens = word_tokeniser_no_lowercase.tokenise("Hello WORLD")
+        assert tokens == ["Hello", "WORLD"]
+
+    def test_punctuation_removal(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("Hello, world! How are you?")
+        assert tokens == ["hello", "world", "how", "are", "you"]
+
+    def test_keep_punctuation(
+        self, word_tokeniser_keep_punctuation: WordTokeniser
+    ) -> None:
+        tokens = word_tokeniser_keep_punctuation.tokenise("Hello, world!")
+        assert tokens == ["hello,", "world!"]
+
+    def test_empty_string(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("")
+        assert tokens == []
+
+    def test_whitespace_only(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("   \t\n  ")
+        assert tokens == []
+
+    def test_multiple_spaces(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("hello    world")
+        assert tokens == ["hello", "world"]
+
+    def test_unicode_nfc_normalisation(self) -> None:
+        # 'é' can be composed (U+00E9) or decomposed (e + U+0301)
+        composed = "caf\u00e9"  # café with composed é
+        decomposed = "cafe\u0301"  # café with decomposed é
+
+        tokeniser = WordTokeniser()
+        tokens_composed = tokeniser.tokenise(composed)
+        tokens_decomposed = tokeniser.tokenise(decomposed)
+
+        # After NFC normalisation, both should be the same
+        assert tokens_composed == tokens_decomposed
+        assert tokens_composed == ["café"]
+
+    def test_unicode_emoji(self, word_tokeniser: WordTokeniser) -> None:
+        # Emoji are removed as punctuation by default
+        tokens = word_tokeniser.tokenise("Hello 👋 world 🌍")
+        assert tokens == ["hello", "world"]
+
+    def test_unicode_non_latin(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("日本語 テスト")
+        assert tokens == ["日本語", "テスト"]
+
+    def test_unicode_mixed_scripts(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("Hello 世界 Bonjour мир")
+        assert tokens == ["hello", "世界", "bonjour", "мир"]
+
+    def test_numbers_preserved(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("I have 42 apples")
+        assert tokens == ["i", "have", "42", "apples"]
+
+    def test_contractions(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("I can't don't won't")
+        # Apostrophe is replaced with space, splitting the words
+        assert tokens == ["i", "can", "t", "don", "t", "won", "t"]
+
+    def test_hyphenated_words(self, word_tokeniser: WordTokeniser) -> None:
+        tokens = word_tokeniser.tokenise("state-of-the-art")
+        # Hyphens are removed as punctuation
+        assert tokens == ["state", "of", "the", "art"]
+
+    def test_custom_normalisation_form(self) -> None:
+        tokeniser = WordTokeniser(normalisation_form="NFKC")
+        # NFKC normalises compatibility characters
+        # ™ (U+2122) is compatibility equivalent to 'TM'
+        tokens = tokeniser.tokenise("test™")
+        assert tokens == ["testtm"]
+
+
+class TestTokeniserProtocol:
+    def test_word_tokeniser_implements_protocol(self) -> None:
+        tokeniser: Tokeniser = WordTokeniser()
+        # Check that it has the required method
+        assert hasattr(tokeniser, "tokenise")
+        assert callable(tokeniser.tokenise)
+        # Check return type
+        result = tokeniser.tokenise("test")
+        assert isinstance(result, list)
+        assert all(isinstance(t, str) for t in result)