From 289cba779506f2c09b0027b830642205a4b4d754 Mon Sep 17 00:00:00 2001
From: cat0825 <1759138827@qq.com>
Date: Sun, 31 May 2026 19:34:50 +0800
Subject: [PATCH 1/3] fix: split CJK from Latin regex tokens

---
 langextract/core/tokenizer.py | 20 ++++++++++++--------
 tests/tokenizer_test.py       | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/langextract/core/tokenizer.py b/langextract/core/tokenizer.py
index a46e5311..74ad0492 100644
--- a/langextract/core/tokenizer.py
+++ b/langextract/core/tokenizer.py
@@ -145,16 +145,22 @@ class TokenizedText:
   tokens: list[Token] = dataclasses.field(default_factory=list)
 
 
-_LETTERS_PATTERN = r"[^\W\d_]+"
+_CJK_CHARS_PATTERN = r"\p{Is_Han}\p{Is_Hiragana}\p{Is_Katakana}\p{Is_Hangul}"
+_LETTERS_PATTERN = rf"[[^\W\d_]--[{_CJK_CHARS_PATTERN}]]+"
 _DIGITS_PATTERN = r"\d+"
+_CJK_PATTERN = rf"[{_CJK_CHARS_PATTERN}]+"
 # Group identical symbols (e.g. "!!") but split mixed ones.
 _SYMBOLS_PATTERN = r"([^\w\s]|_)\1*"
 _END_OF_SENTENCE_PATTERN = regex.compile(r"[.?!。！？\u0964][\"'”’»)\]}]*$")
 
 _TOKEN_PATTERN = regex.compile(
-    rf"{_LETTERS_PATTERN}|{_DIGITS_PATTERN}|{_SYMBOLS_PATTERN}"
+    rf"{_CJK_PATTERN}|{_LETTERS_PATTERN}|{_DIGITS_PATTERN}|{_SYMBOLS_PATTERN}",
+    flags=regex.V1,
+)
+_WORD_PATTERN = regex.compile(
+    rf"(?:{_CJK_PATTERN}|{_LETTERS_PATTERN}|{_DIGITS_PATTERN})\Z",
+    flags=regex.V1,
 )
-_WORD_PATTERN = regex.compile(rf"(?:{_LETTERS_PATTERN}|{_DIGITS_PATTERN})\Z")
 
 # Abbreviations that do not end sentences.
 # TODO: Evaluate removal for large-context use cases.
@@ -246,9 +252,7 @@ def tokenize(
   return tokenizer.tokenize(text)
 
 
-_CJK_PATTERN = regex.compile(
-    r"\p{Is_Han}|\p{Is_Hiragana}|\p{Is_Katakana}|\p{Is_Hangul}"
-)
+_CJK_REGEX = regex.compile(_CJK_PATTERN)
 _NON_SPACED_PATTERN = regex.compile(
     r"\p{Is_Thai}|\p{Is_Lao}|\p{Is_Khmer}|\p{Is_Myanmar}"
 )
@@ -380,7 +384,7 @@ def tokenize(self, text: str) -> TokenizedText:
               should_merge = False
 
             # CJK and Non-Spaced scripts require fragmentation.
-            elif _CJK_PATTERN.match(first_char) or _NON_SPACED_PATTERN.match(
+            elif _CJK_REGEX.match(first_char) or _NON_SPACED_PATTERN.match(
                 first_char
             ):
               should_merge = False
@@ -426,7 +430,7 @@ def tokenize(self, text: str) -> TokenizedText:
         # Determine script for the new token
         if current_type == TokenType.WORD:
           c = grapheme[0]
-          if _CJK_PATTERN.match(c) or _NON_SPACED_PATTERN.match(c):
+          if _CJK_REGEX.match(c) or _NON_SPACED_PATTERN.match(c):
             current_script = _NO_GROUP_SCRIPT
           else:
             current_script = _get_script_fast(c)
diff --git a/tests/tokenizer_test.py b/tests/tokenizer_test.py
index 11914ae2..378b42c5 100644
--- a/tests/tokenizer_test.py
+++ b/tests/tokenizer_test.py
@@ -224,6 +224,39 @@ def test_underscore_handling(self):
     )
     self.assertEqual(tokenized.tokens[2].token_type, tokenizer.TokenType.WORD)
 
+  @parameterized.named_parameters(
+      dict(
+          testcase_name="latin_cjk_boundary",
+          input_text="Hello世界",
+          expected_tokens=[
+              ("Hello", tokenizer.TokenType.WORD),
+              ("世界", tokenizer.TokenType.WORD),
+          ],
+      ),
+      dict(
+          testcase_name="accented_latin_cjk_boundary",
+          input_text="café世界",
+          expected_tokens=[
+              ("café", tokenizer.TokenType.WORD),
+              ("世界", tokenizer.TokenType.WORD),
+          ],
+      ),
+  )
+  def test_regex_tokenizer_splits_cjk_from_latin_words(
+      self, input_text, expected_tokens
+  ):
+    tokenized = tokenizer.RegexTokenizer().tokenize(input_text)
+    actual_tokens = [
+        (
+            input_text[
+                token.char_interval.start_pos : token.char_interval.end_pos
+            ],
+            token.token_type,
+        )
+        for token in tokenized.tokens
+    ]
+    self.assertEqual(actual_tokens, expected_tokens)
+
 
 class UnicodeTokenizerTest(parameterized.TestCase):
   # pylint: disable=too-many-public-methods

From cbd36e6d80c13568d3ade513bf3db6267d64bd34 Mon Sep 17 00:00:00 2001
From: cat0825 <1759138827@qq.com>
Date: Thu, 11 Jun 2026 23:11:10 +0800
Subject: [PATCH 2/3] test: cover Hangul/Latin and Hangul/digit token
 boundaries

---
 tests/tokenizer_test.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tests/tokenizer_test.py b/tests/tokenizer_test.py
index 378b42c5..842158c1 100644
--- a/tests/tokenizer_test.py
+++ b/tests/tokenizer_test.py
@@ -241,6 +241,22 @@ def test_underscore_handling(self):
               ("世界", tokenizer.TokenType.WORD),
           ],
       ),
+      dict(
+          testcase_name="latin_hangul_boundary",
+          input_text="Minsoo는",
+          expected_tokens=[
+              ("Minsoo", tokenizer.TokenType.WORD),
+              ("는", tokenizer.TokenType.WORD),
+          ],
+      ),
+      dict(
+          testcase_name="digit_hangul_boundary",
+          input_text="9900원",
+          expected_tokens=[
+              ("9900", tokenizer.TokenType.NUMBER),
+              ("원", tokenizer.TokenType.WORD),
+          ],
+      ),
   )
   def test_regex_tokenizer_splits_cjk_from_latin_words(
       self, input_text, expected_tokens

From 3c7dedadfaee50aadee59ab4dc839dc74cade2c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C4=81na=28Bass=20Ver=2E=29?= <1759138827@qq.com>
Date: Fri, 12 Jun 2026 11:56:50 +0800
Subject: [PATCH 3/3] chore: rerun community support check