wordPieces = new ArrayList<>();
+ int start = from;
+ boolean found = true;
+ while (start < to) {
+ int end = to;
+ found = false;
+ while (start < end) {
+ String substring = new String(mapped.chars, start, end - start);
+ if (start > from) {
+ substring = CONTINUATION_PREFIX + substring;
+ }
+ if (vocabulary.contains(substring)) {
+ wordPieces.add(new SubwordPiece(substring, ids.get(substring),
+ mapped.starts[start], mapped.ends[end - 1]));
+ start = end;
+ found = true;
+ break;
+ }
+ end--;
+ }
+ if (!found) {
+ break;
+ }
+ }
+ if (found) {
+ pieces.addAll(wordPieces);
+ } else {
+ pieces.add(new SubwordPiece(unknownToken, unknownId, wordStart, wordEnd));
+ }
+ }
+
+ /**
+ * The normalized text with, for every char, the original-text range it came from. Characters
+ * inserted by the pipeline (isolation spaces) carry an empty range at the insertion point.
+ */
+ private static final class MappedText {
+ private char[] chars;
+ private int[] starts;
+ private int[] ends;
+ private int length;
+
+ /**
+ * Instantiates an empty mapped text.
+ *
+ * @param capacity The initial capacity hint in chars.
+ */
+ private MappedText(int capacity) {
+ chars = new char[capacity];
+ starts = new int[capacity];
+ ends = new int[capacity];
+ }
+
+ /**
+ * Appends one char with the original-text range it came from.
+ *
+ * @param c The char to append.
+ * @param originalStart The inclusive original-text start of the char.
+ * @param originalEnd The exclusive original-text end of the char.
+ */
+ private void add(char c, int originalStart, int originalEnd) {
+ if (length == chars.length) {
+ final int capacity = Math.max(16, length * 2);
+ chars = Arrays.copyOf(chars, capacity);
+ starts = Arrays.copyOf(starts, capacity);
+ ends = Arrays.copyOf(ends, capacity);
+ }
+ chars[length] = c;
+ starts[length] = originalStart;
+ ends[length] = originalEnd;
+ length++;
+ }
+
+ /**
+ * Appends every char of a string, all sharing one original-text range.
+ *
+ * @param s The string to append.
+ * @param originalStart The inclusive original-text start shared by all chars.
+ * @param originalEnd The exclusive original-text end shared by all chars.
+ */
+ private void add(String s, int originalStart, int originalEnd) {
+ for (int i = 0; i < s.length(); i++) {
+ add(s.charAt(i), originalStart, originalEnd);
+ }
+ }
+ }
+
+ /**
+ * Cleans the text (control and whitespace normalization) and isolates CJK code points in one
+ * pass, recording the original-text range of every output character.
+ *
+ * @param original The original input text.
+ * @return The cleaned, CJK-isolated text with per-character ranges.
+ */
+ private static MappedText cleanAndIsolateCjk(String original) {
+ final MappedText out = new MappedText(original.length() + 16);
+ int i = 0;
+ while (i < original.length()) {
+ final int codePoint = original.codePointAt(i);
+ final int width = Character.charCount(codePoint);
+ if (codePoint == 0 || codePoint == 0xFFFD || BertNormalization.isControl(codePoint)) {
+ i += width;
+ continue;
+ }
+ if (BertNormalization.isWhitespace(codePoint) || isLineOrParagraphSeparator(codePoint)) {
+ out.add(' ', i, i + width);
+ } else if (BertNormalization.isCjk(codePoint)) {
+ out.add(' ', i, i);
+ for (int c = 0; c < width; c++) {
+ out.add(original.charAt(i + c), i, i + width);
+ }
+ out.add(' ', i + width, i + width);
+ } else {
+ for (int c = 0; c < width; c++) {
+ out.add(original.charAt(i + c), i, i + width);
+ }
+ }
+ i += width;
+ }
+ return out;
+ }
+
+ /**
+ * A Unicode line or paragraph separator ({@code Zl}, {@code Zp}). These are not whitespace in
+ * the BERT {@code _is_whitespace} sense, but the reference pipeline's
+ * {@code whitespace_tokenize} (Python's {@code str.split()}) still breaks words on them, so
+ * they must become word boundaries here as well.
+ *
+ * @param codePoint The code point to classify.
+ * @return True if the code point is a line or paragraph separator.
+ */
+ private static boolean isLineOrParagraphSeparator(int codePoint) {
+ final int type = Character.getType(codePoint);
+ return type == Character.LINE_SEPARATOR || type == Character.PARAGRAPH_SEPARATOR;
+ }
+
+ /**
+ * Isolates punctuation, surrounding each punctuation code point with spaces, preserving the
+ * original-text range of every character.
+ *
+ * @param in The input text with per-character ranges.
+ * @return The punctuation-isolated text with per-character ranges.
+ */
+ private static MappedText isolatePunctuation(MappedText in) {
+ final MappedText out = new MappedText(in.length + 16);
+ int i = 0;
+ while (i < in.length) {
+ final int codePoint = codePointAt(in, i);
+ final int width = Character.charCount(codePoint);
+ if (BertNormalization.isPunctuation(codePoint)) {
+ out.add(' ', in.starts[i], in.starts[i]);
+ for (int c = 0; c < width; c++) {
+ out.add(in.chars[i + c], in.starts[i + c], in.ends[i + c]);
+ }
+ out.add(' ', in.ends[i + width - 1], in.ends[i + width - 1]);
+ } else {
+ for (int c = 0; c < width; c++) {
+ out.add(in.chars[i + c], in.starts[i + c], in.ends[i + c]);
+ }
+ }
+ i += width;
+ }
+ return out;
+ }
+
+ /**
+ * Lower cases and strips accents, preserving the original-text range of every character. When a
+ * contextual case mapping prevents a per-character range from being recovered, the whole
+ * whitespace run falls back to its full range, which widens spans but never misplaces them.
+ *
+ * @param in The input text with per-character ranges.
+ * @return The lower-cased, accent-stripped text with per-character ranges.
+ */
+ private static MappedText lowerCaseAndStripAccents(MappedText in) {
+ final MappedText out = new MappedText(in.length + 16);
+ int from = 0;
+ while (from < in.length) {
+ if (in.chars[from] == ' ') {
+ out.add(' ', in.starts[from], in.ends[from]);
+ from++;
+ continue;
+ }
+ int to = from;
+ while (to < in.length && in.chars[to] != ' ') {
+ to++;
+ }
+ transformRun(in, from, to, out);
+ from = to;
+ }
+ return out;
+ }
+
+ /**
+ * Lower cases and accent-strips one non-space run, emitting per-character ranges when the
+ * transformation is reproducible per code point and the run's full range otherwise.
+ *
+ * @param in The input text with per-character ranges.
+ * @param from The inclusive start of the run in {@code in}.
+ * @param to The exclusive end of the run in {@code in}.
+ * @param out The output text to append to.
+ */
+ private static void transformRun(MappedText in, int from, int to, MappedText out) {
+ final String run = new String(in.chars, from, to - from);
+ // Locale.ROOT lower casing is the reference behavior of BERT's do_lower_case: the reference
+ // pipeline applies the full locale-independent Unicode case mappings (including one-to-many
+ // ones like the dotted capital I), which a per-code-point mapping cannot reproduce.
+ final String content = stripAccents(run.toLowerCase(Locale.ROOT));
+
+ // Rerun per code point to learn how many output chars each input code point produces.
+ final StringBuilder rerun = new StringBuilder(content.length());
+ final int[] produced = new int[to - from];
+ int i = from;
+ while (i < to) {
+ final int codePoint = codePointAt(in, i);
+ final int width = Character.charCount(codePoint);
+ final String transformed = stripAccents(
+ new String(Character.toChars(codePoint)).toLowerCase(Locale.ROOT));
+ rerun.append(transformed);
+ produced[i - from] = transformed.length();
+ i += width;
+ }
+
+ if (rerun.toString().equals(content)) {
+ int at = from;
+ int emitted = 0;
+ while (at < to) {
+ final int width = Character.charCount(codePointAt(in, at));
+ for (int c = 0; c < produced[at - from]; c++) {
+ out.add(content.charAt(emitted++), in.starts[at], in.ends[at + width - 1]);
+ }
+ at += width;
+ }
+ } else {
+ // Contextual case mapping changed the content; the run's chars share the run's range.
+ out.add(content, in.starts[from], in.ends[to - 1]);
+ }
+ }
+
+ /**
+ * Removes combining marks after NFD decomposition, the accent stripping of BERT's
+ * {@code do_lower_case} mode.
+ *
+ * @param text The text to strip.
+ * @return The text without non-spacing marks.
+ */
+ private static String stripAccents(String text) {
+ final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD);
+ final StringBuilder stripped = new StringBuilder(decomposed.length());
+ decomposed.codePoints().forEach(codePoint -> {
+ if (Character.getType(codePoint) != Character.NON_SPACING_MARK) {
+ stripped.appendCodePoint(codePoint);
+ }
+ });
+ return stripped.toString();
+ }
+
+ /**
+ * Reads the code point at an index, joining a surrogate pair when one starts there.
+ *
+ * @param text The text to read from.
+ * @param index The char index to read at.
+ * @return The code point at {@code index}.
+ */
+ private static int codePointAt(MappedText text, int index) {
+ final char c = text.chars[index];
+ if (Character.isHighSurrogate(c) && index + 1 < text.length
+ && Character.isLowSurrogate(text.chars[index + 1])) {
+ return Character.toCodePoint(c, text.chars[index + 1]);
+ }
+ return c;
+ }
+}
diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java
index 97e240e87c..fa1014ab2c 100644
--- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java
+++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java
@@ -34,7 +34,7 @@
* of BERT tokenization. It does not normalize the input text: no lower casing,
* no accent stripping, no control character removal. Text that does not match
* the vocabulary's casing - for uncased models that includes every capitalized
- * word - is mapped to the unknown token. Use {@link BertTokenizer} for the
+ * word - is mapped to the unknown token. Use {@link WordpieceEncoder} for the
* full BERT tokenization pipeline.
*
* As of OpenNLP 3.0.0 the behavior matches the reference BERT wordpiece
@@ -58,7 +58,7 @@
*
*
*
- * @see BertTokenizer
+ * @see WordpieceEncoder
*/
public class WordpieceTokenizer implements Tokenizer {
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
index a598199270..8f59c6f844 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java
@@ -105,7 +105,7 @@ protected AbstractDL(final File model, final File vocabulary,
final OrtSession createdSession = env.createSession(model.getPath(), sessionOptions);
try {
this.vocab = Map.copyOf(loadVocabFile(vocabulary));
- this.tokenizer = createBertTokenizer(vocab, lowerCase);
+ this.tokenizer = createPipelineTokenizer(vocab, lowerCase);
} catch (IOException | RuntimeException e) {
// Vocabulary/tokenizer init failed after the native session was created; close it
// so a partially constructed instance never leaks the ONNX session.
@@ -136,7 +136,7 @@ protected AbstractDL(final OrtEnvironment env, final OrtSession session,
this.env = env;
this.session = session;
this.vocab = vocab;
- this.tokenizer = createBertTokenizer(vocab, lowerCase);
+ this.tokenizer = createPipelineTokenizer(vocab, lowerCase);
}
/**
@@ -247,16 +247,31 @@ static WordpieceTokenizer createWordpieceTokenizer(
* @param lowerCase {@code true} for uncased models (lower casing and accent
* stripping), {@code false} for cased models.
* @return A configured {@link BertTokenizer}.
- * @throws IllegalArgumentException Thrown if a RoBERTa-style vocabulary
- * contains no supported unknown token.
+ * @throws IllegalArgumentException Thrown if the selected special tokens
+ * are not all present in the vocabulary.
*/
+ // The deprecated BertTokenizer stays the return type until its removal in 3.1, so that
+ // already-compiled subclasses overriding this method keep overriding it.
+ @SuppressWarnings("removal")
protected BertTokenizer createTokenizer(
final Map vocab, final boolean lowerCase) {
- return createBertTokenizer(vocab, lowerCase);
+ return createPipelineTokenizer(vocab, lowerCase);
}
- static BertTokenizer createBertTokenizer(
+ /**
+ * Builds the pipeline tokenizer, selecting the RoBERTa special tokens when the vocabulary
+ * carries them and the BERT defaults otherwise.
+ *
+ * @param vocab The vocabulary map.
+ * @param lowerCase {@code true} for uncased models, {@code false} for cased models.
+ * @return A configured {@link BertTokenizer}.
+ * @throws IllegalArgumentException Thrown if the selected special tokens are not all present in
+ * the vocabulary.
+ */
+ // BertTokenizer is deprecated for removal in 3.1; built here until then.
+ @SuppressWarnings("removal")
+ static BertTokenizer createPipelineTokenizer(
final Map vocab, final boolean lowerCase) {
if (vocab.containsKey(
WordpieceTokenizer.ROBERTA_CLS_TOKEN)
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java
index e5fbe2c6a4..7b17cb3028 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java
@@ -51,7 +51,7 @@
* using ONNX models.
*
* Tokenization performs BERT basic tokenization (text normalization)
- * before wordpiece, see {@link opennlp.tools.tokenize.BertTokenizer}. Input
+ * before wordpiece, see {@link opennlp.tools.tokenize.WordpieceEncoder}. Input
* text is lower cased and accent stripped by default, matching the uncased
* models commonly used for classification. For cased models, set
* {@link InferenceOptions#setLowerCase(boolean)} to {@code false}.
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java
index 95bb81d879..956f177d61 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java
@@ -50,7 +50,7 @@
* An implementation of {@link opennlp.tools.namefind.TokenNameFinder} that uses ONNX models.
*
* Tokenization performs BERT basic tokenization (text normalization)
- * before wordpiece, see {@link opennlp.tools.tokenize.BertTokenizer}. Input
+ * before wordpiece, see {@link opennlp.tools.tokenize.WordpieceEncoder}. Input
* text is not lower cased by default, because named entity recognition
* models are commonly cased: capitalization is a strong signal for entity
* boundaries. For uncased models, set
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java
index 6bc76ce183..f1250ea601 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java
@@ -47,7 +47,7 @@
* so the encoder attended to nothing and the output vectors were
* incorrect. Additionally, tokenization now performs BERT basic
* tokenization (lower casing and accent stripping by default, see
- * {@link opennlp.tools.tokenize.BertTokenizer}) before wordpiece.
+ * {@link opennlp.tools.tokenize.WordpieceEncoder}) before wordpiece.
* Output vectors change with the corrected encoding and tokenization;
* any embeddings persisted from the previous behavior are not
* comparable with the corrected output and must be re-embedded.
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java
index 54c4600a8e..5ae7e3ebee 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java
@@ -22,7 +22,7 @@
import org.junit.jupiter.api.Test;
-import opennlp.tools.tokenize.BertTokenizer;
+import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.WordpieceTokenizer;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
@@ -52,8 +52,8 @@ private static Map robertaVocab() {
}
@Test
- void testCreatesLowerCasingBertTokenizer() {
- final BertTokenizer tokenizer = AbstractDL.createBertTokenizer(bertVocab(), true);
+ void testCreatesLowerCasingPipelineTokenizer() {
+ final Tokenizer tokenizer = AbstractDL.createPipelineTokenizer(bertVocab(), true);
// Capitalized input must be lower cased before the wordpiece lookup.
assertArrayEquals(new String[] {
@@ -62,8 +62,8 @@ void testCreatesLowerCasingBertTokenizer() {
}
@Test
- void testCreatesCasePreservingBertTokenizer() {
- final BertTokenizer tokenizer = AbstractDL.createBertTokenizer(bertVocab(), false);
+ void testCreatesCasePreservingPipelineTokenizer() {
+ final Tokenizer tokenizer = AbstractDL.createPipelineTokenizer(bertVocab(), false);
// Without lower casing, capitalized words miss the lowercase-only vocabulary.
assertArrayEquals(new String[] {
@@ -74,7 +74,7 @@ void testCreatesCasePreservingBertTokenizer() {
@Test
void testSelectsRobertaSpecialTokens() {
- final BertTokenizer tokenizer = AbstractDL.createBertTokenizer(robertaVocab(), false);
+ final Tokenizer tokenizer = AbstractDL.createPipelineTokenizer(robertaVocab(), false);
assertArrayEquals(new String[] {
WordpieceTokenizer.ROBERTA_CLS_TOKEN, "hello", WordpieceTokenizer.ROBERTA_UNK_TOKEN,
@@ -88,7 +88,7 @@ void testFallsBackToBertUnknownToken() {
vocab.remove(WordpieceTokenizer.ROBERTA_UNK_TOKEN);
vocab.put(WordpieceTokenizer.BERT_UNK_TOKEN, 2);
- final BertTokenizer tokenizer = AbstractDL.createBertTokenizer(vocab, false);
+ final Tokenizer tokenizer = AbstractDL.createPipelineTokenizer(vocab, false);
assertArrayEquals(new String[] {
WordpieceTokenizer.ROBERTA_CLS_TOKEN, "hello", WordpieceTokenizer.BERT_UNK_TOKEN,
@@ -101,10 +101,26 @@ void testRejectsRobertaVocabularyWithoutUnknownToken() {
final Map vocab = robertaVocab();
vocab.remove(WordpieceTokenizer.ROBERTA_UNK_TOKEN);
- assertThrows(IllegalArgumentException.class, () -> AbstractDL.createBertTokenizer(vocab, false));
+ assertThrows(IllegalArgumentException.class,
+ () -> AbstractDL.createPipelineTokenizer(vocab, false));
assertThrows(IllegalArgumentException.class, () -> AbstractDL.createWordpieceTokenizer(vocab));
}
+ @Test
+ void testTokenizePosIsUnsupported() {
+ final Tokenizer tokenizer = AbstractDL.createPipelineTokenizer(bertVocab(), true);
+ assertThrows(UnsupportedOperationException.class, () -> tokenizer.tokenizePos("the fox"));
+ }
+
+ @Test
+ void testRejectsBertVocabularyMissingSpecialTokensAtCreation() {
+ final Map vocab = bertVocab();
+ vocab.remove(WordpieceTokenizer.BERT_UNK_TOKEN);
+
+ assertThrows(IllegalArgumentException.class,
+ () -> AbstractDL.createPipelineTokenizer(vocab, true));
+ }
+
@Test
void testResolveLowerCaseUsesComponentDefaultWhenUnset() {
final InferenceOptions options = new InferenceOptions();
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java
index d8f706f4ef..070834316b 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java
@@ -14,149 +14,89 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package opennlp.tools.tokenize;
+import java.util.HashSet;
+import java.util.List;
import java.util.Set;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
/**
- * Tests {@link BertTokenizer}.
- *
- * All expected token sequences in this test were generated with the HuggingFace
- * {@code tokenizers} reference implementation ({@code BertWordPieceTokenizer})
- * using the same vocabulary, so they are verified to be identical to the
- * reference BERT tokenization.
+ * Pins the deprecated {@link BertTokenizer} shim: constructor validation, delegation to
+ * {@link WordpieceEncoder#encodeToPieces(CharSequence)}, the default special tokens, and
+ * the unsupported {@link BertTokenizer#tokenizePos(String)}.
*/
-public class BertTokenizerTest {
-
- private static final Set VOCABULARY = Set.of(
- "the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog",
- "em", "##bed", "##ding", "##s",
- "wurttemberg", "strasse", "grosse",
- "don", "t", "wait", "what", ".", ",", "?", "!", "'",
- "\u6211", "\u7231", // CJK: 我 爱
- "natural", "language", "processing");
-
- @Test
- void testLowerCasesCapitalizedWords() {
- final Tokenizer tokenizer = new BertTokenizer(VOCABULARY);
- final String[] tokens = tokenizer.tokenize("The quick brown fox jumps over the lazy dog.");
-
- final String[] expected = {"[CLS]", "the", "quick", "brown", "fox", "jumps", "over",
- "the", "lazy", "dog", ".", "[SEP]"};
- Assertions.assertArrayEquals(expected, tokens);
- }
-
- @Test
- void testLowerCasesBeforeWordpieceSplitting() {
- final Tokenizer tokenizer = new BertTokenizer(VOCABULARY);
- final String[] tokens = tokenizer.tokenize("Embeddings");
-
- final String[] expected = {"[CLS]", "em", "##bed", "##ding", "##s", "[SEP]"};
- Assertions.assertArrayEquals(expected, tokens);
- }
-
- @Test
- void testStripsAccentsButKeepsNonCombiningCharacters() {
- final Tokenizer tokenizer = new BertTokenizer(VOCABULARY);
- // ü decomposes to u + combining diaeresis and the mark is stripped;
- // ß is not a combining mark and must survive, leaving an OOV token.
- final String[] tokens = tokenizer.tokenize("W\u00fcrttemberg Stra\u00dfe");
-
- final String[] expected = {"[CLS]", "wurttemberg", "[UNK]", "[SEP]"};
- Assertions.assertArrayEquals(expected, tokens);
- }
+@SuppressWarnings("removal") // Exercises BertTokenizer deliberately until its removal in 3.1.
+class BertTokenizerTest {
- @Test
- void testSplitsPunctuationRunsIntoSingleCharacters() {
- final Tokenizer tokenizer = new BertTokenizer(VOCABULARY);
- final String[] tokens = tokenizer.tokenize("Wait... what?!");
+ private static final List VOCABULARY = List.of(
+ "[PAD]", "[UNK]", "[CLS]", "[SEP]", "hello", "world", "##s", "ca", "##fe", ",", "!");
- final String[] expected = {"[CLS]", "wait", ".", ".", ".", "what", "?", "!", "[SEP]"};
- Assertions.assertArrayEquals(expected, tokens);
+ private static Set vocabularySet() {
+ return new HashSet<>(VOCABULARY);
}
- @Test
- void testSplitsApostrophesAsPunctuation() {
- final Tokenizer tokenizer = new BertTokenizer(VOCABULARY);
- final String[] tokens = tokenizer.tokenize("don't");
-
- final String[] expected = {"[CLS]", "don", "'", "t", "[SEP]"};
- Assertions.assertArrayEquals(expected, tokens);
+ @ParameterizedTest
+ @ValueSource(strings = {"Hello, world!", "Caf\u00E9 hellos", ""})
+ void testTokenizeReturnsTheEncoderPieceSequence(String input) {
+ final BertTokenizer tokenizer = new BertTokenizer(vocabularySet());
+ final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY);
+ assertArrayEquals(encoder.encodeToPieces(input), tokenizer.tokenize(input),
+ "delegation broke on: " + input);
}
@Test
- void testIsolatesCjkIdeographs() {
- final Tokenizer tokenizer = new BertTokenizer(VOCABULARY);
- final String[] tokens = tokenizer.tokenize("\u6211\u7231natural language processing");
-
- final String[] expected = {"[CLS]", "\u6211", "\u7231", "natural", "language",
- "processing", "[SEP]"};
- Assertions.assertArrayEquals(expected, tokens);
+ void testDefaultConstructorsChainToBertSpecialTokensAndLowerCasing() {
+ final String[] uncasedDefault = new BertTokenizer(vocabularySet()).tokenize("Hello worldS");
+ assertArrayEquals(new String[] {"[CLS]", "hello", "world", "##s", "[SEP]"}, uncasedDefault);
+ assertArrayEquals(uncasedDefault,
+ new BertTokenizer(vocabularySet(), true).tokenize("Hello worldS"));
+ assertArrayEquals(uncasedDefault,
+ new BertTokenizer(vocabularySet(), true, WordpieceTokenizer.BERT_CLS_TOKEN,
+ WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN)
+ .tokenize("Hello worldS"));
}
@Test
- void testCleansControlCharactersAndNormalizesWhitespace() {
- final Tokenizer tokenizer = new BertTokenizer(VOCABULARY);
- // Tab and no-break space are whitespace; the NUL character is removed,
- // joining "brown" and "fox" into one out-of-vocabulary token.
- final String[] tokens = tokenizer.tokenize("the\tquick\u00a0brown\u0000fox");
-
- final String[] expected = {"[CLS]", "the", "quick", "[UNK]", "[SEP]"};
- Assertions.assertArrayEquals(expected, tokens);
+ void testCasedTokenizerKeepsCase() {
+ // Without lower casing, the capitalized word misses the lowercase-only vocabulary.
+ assertArrayEquals(new String[] {"[CLS]", "[UNK]", "[SEP]"},
+ new BertTokenizer(vocabularySet(), false).tokenize("Hello"));
}
@Test
- void testRemovesPrivateUseAndUnassignedCharacters() {
- final Tokenizer tokenizer = new BertTokenizer(VOCABULARY);
- // The reference implementation treats all C* categories as control
- // characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn)
- // are removed, joining the surrounding text into one OOV token.
- final String[] tokens = tokenizer.tokenize("fox\ue000jumps and fox\ufdd0jumps");
-
- final String[] expected = {"[CLS]", "[UNK]", "[UNK]", "[UNK]", "[SEP]"};
- Assertions.assertArrayEquals(expected, tokens);
+ void testConstructorsRejectNullArguments() {
+ assertThrows(IllegalArgumentException.class, () -> new BertTokenizer(null));
+ assertThrows(IllegalArgumentException.class, () -> new BertTokenizer(null, true));
+ assertThrows(IllegalArgumentException.class,
+ () -> new BertTokenizer(null, true, "[CLS]", "[SEP]", "[UNK]"));
+ assertThrows(IllegalArgumentException.class,
+ () -> new BertTokenizer(vocabularySet(), true, null, "[SEP]", "[UNK]"));
+ assertThrows(IllegalArgumentException.class,
+ () -> new BertTokenizer(vocabularySet(), true, "[CLS]", null, "[UNK]"));
+ assertThrows(IllegalArgumentException.class,
+ () -> new BertTokenizer(vocabularySet(), true, "[CLS]", "[SEP]", null));
}
@Test
- void testRejectsNullSpecialTokens() {
- Assertions.assertThrows(NullPointerException.class,
- () -> new BertTokenizer(VOCABULARY, true, null, "[SEP]", "[UNK]"));
- Assertions.assertThrows(NullPointerException.class,
- () -> new BertTokenizer(VOCABULARY, true, "[CLS]", null, "[UNK]"));
- Assertions.assertThrows(NullPointerException.class,
- () -> new BertTokenizer(VOCABULARY, true, "[CLS]", "[SEP]", null));
+ void testTokenizeRejectsNullText() {
+ final BertTokenizer tokenizer = new BertTokenizer(vocabularySet());
+ assertThrows(IllegalArgumentException.class, () -> tokenizer.tokenize(null));
}
@Test
- void testCasedModeKeepsCaseAndAccents() {
- final Tokenizer tokenizer = new BertTokenizer(
- Set.of("The", "W\u00fcrttemberg", "fox"), false);
- final String[] tokens = tokenizer.tokenize("The W\u00fcrttemberg fox");
-
- final String[] expected = {"[CLS]", "The", "W\u00fcrttemberg", "fox", "[SEP]"};
- Assertions.assertArrayEquals(expected, tokens);
+ void testTokenizePosIsUnsupportedWithTheDocumentedMessage() {
+ final BertTokenizer tokenizer = new BertTokenizer(vocabularySet());
+ final UnsupportedOperationException e = assertThrows(UnsupportedOperationException.class,
+ () -> tokenizer.tokenizePos("hello world"));
+ assertEquals("Wordpiece tokens cannot be mapped to character spans of the original text",
+ e.getMessage());
}
-
- @Test
- void testCustomSpecialTokens() {
- final Tokenizer tokenizer = new BertTokenizer(Set.of("the", "fox"), true,
- WordpieceTokenizer.ROBERTA_CLS_TOKEN, WordpieceTokenizer.ROBERTA_SEP_TOKEN,
- WordpieceTokenizer.ROBERTA_UNK_TOKEN);
- final String[] tokens = tokenizer.tokenize("The unknown fox");
-
- final String[] expected = {"", "the", "", "fox", ""};
- Assertions.assertArrayEquals(expected, tokens);
- }
-
- @Test
- void testTokenizePosIsUnsupported() {
- final Tokenizer tokenizer = new BertTokenizer(VOCABULARY);
- Assertions.assertThrows(UnsupportedOperationException.class,
- () -> tokenizer.tokenizePos("the fox"));
- }
-
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java
new file mode 100644
index 0000000000..064d644a57
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.tokenize;
+
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+/**
+ * Reference token-sequence expectations for {@link WordpieceEncoder}, covering lower casing,
+ * accent stripping, punctuation and CJK isolation, and text cleaning.
+ *
+ * All expected token sequences in this test were generated with the HuggingFace
+ * {@code tokenizers} reference implementation ({@code BertWordPieceTokenizer})
+ * using the same vocabulary, so they are verified to be identical to the
+ * reference BERT tokenization. The encoder requires its special tokens to be
+ * present in the vocabulary (every piece must have an id), so the vocabularies
+ * here include them; the token sequences are unchanged.
+ */
+public class WordpieceEncoderReferenceSequencesTest {
+
+ private static final List VOCABULARY = List.of(
+ "[CLS]", "[SEP]", "[UNK]",
+ "the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog",
+ "em", "##bed", "##ding", "##s",
+ "wurttemberg", "strasse", "grosse",
+ "don", "t", "wait", "what", ".", ",", "?", "!", "'",
+ "\u6211", "\u7231", // CJK
+ "natural", "language", "processing");
+
+ /**
+ * The reference input and expected-sequence pairs, one argument set per pipeline behavior.
+ *
+ * @return The (input, expected pieces) pairs.
+ */
+ static Stream referenceSequences() {
+ return Stream.of(
+ // Lower cases capitalized words.
+ Arguments.of("The quick brown fox jumps over the lazy dog.",
+ new String[] {"[CLS]", "the", "quick", "brown", "fox", "jumps", "over",
+ "the", "lazy", "dog", ".", "[SEP]"}),
+ // Lower cases before wordpiece splitting.
+ Arguments.of("Embeddings",
+ new String[] {"[CLS]", "em", "##bed", "##ding", "##s", "[SEP]"}),
+ // The u-umlaut decomposes to u plus a combining diaeresis and the mark is stripped;
+ // the sharp s is not a combining mark and must survive, leaving an OOV token.
+ Arguments.of("W\u00fcrttemberg Stra\u00dfe",
+ new String[] {"[CLS]", "wurttemberg", "[UNK]", "[SEP]"}),
+ // Splits punctuation runs into single characters.
+ Arguments.of("Wait... what?!",
+ new String[] {"[CLS]", "wait", ".", ".", ".", "what", "?", "!", "[SEP]"}),
+ // Splits apostrophes as punctuation.
+ Arguments.of("don't",
+ new String[] {"[CLS]", "don", "'", "t", "[SEP]"}),
+ // Isolates CJK ideographs into single-character pieces.
+ Arguments.of("\u6211\u7231natural language processing",
+ new String[] {"[CLS]", "\u6211", "\u7231", "natural", "language",
+ "processing", "[SEP]"}),
+ // Tab and no-break space are whitespace; the NUL character is removed,
+ // joining "brown" and "fox" into one out-of-vocabulary token.
+ Arguments.of("the\tquick\u00a0brown\u0000fox",
+ new String[] {"[CLS]", "the", "quick", "[UNK]", "[SEP]"}),
+ // The reference implementation treats all C* categories as control
+ // characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn)
+ // are removed, joining the surrounding text into one OOV token.
+ Arguments.of("fox\ue000jumps and fox\ufdd0jumps",
+ new String[] {"[CLS]", "[UNK]", "[UNK]", "[UNK]", "[SEP]"}));
+ }
+
+ @ParameterizedTest
+ @MethodSource("referenceSequences")
+ void testEncodesTheReferenceSequence(String input, String[] expected) {
+ final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY);
+ Assertions.assertArrayEquals(expected, encoder.encodeToPieces(input),
+ "sequence broke on: " + input);
+ }
+
+ @Test
+ void testRejectsNullSpecialTokens() {
+ // The encoder's contract throws IllegalArgumentException for null special tokens.
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new WordpieceEncoder(VOCABULARY, true, null, "[SEP]", "[UNK]"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new WordpieceEncoder(VOCABULARY, true, "[CLS]", null, "[UNK]"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new WordpieceEncoder(VOCABULARY, true, "[CLS]", "[SEP]", null));
+ }
+
+ @Test
+ void testCasedModeKeepsCaseAndAccents() {
+ final WordpieceEncoder encoder = new WordpieceEncoder(
+ List.of("[CLS]", "[SEP]", "[UNK]", "The", "W\u00fcrttemberg", "fox"), false);
+ final String[] tokens = encoder.encodeToPieces("The W\u00fcrttemberg fox");
+
+ final String[] expected = {"[CLS]", "The", "W\u00fcrttemberg", "fox", "[SEP]"};
+ Assertions.assertArrayEquals(expected, tokens);
+ }
+
+ @Test
+ void testCustomSpecialTokens() {
+ final WordpieceEncoder encoder = new WordpieceEncoder(
+ List.of("", "", "", "the", "fox"), true,
+ WordpieceTokenizer.ROBERTA_CLS_TOKEN, WordpieceTokenizer.ROBERTA_SEP_TOKEN,
+ WordpieceTokenizer.ROBERTA_UNK_TOKEN);
+ final String[] tokens = encoder.encodeToPieces("The unknown fox");
+
+ final String[] expected = {"", "the", "", "fox", ""};
+ Assertions.assertArrayEquals(expected, tokens);
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java
new file mode 100644
index 0000000000..997036b092
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.tokenize;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.util.Span;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The encoder held against the deprecated {@link BertTokenizer} for piece-sequence parity (the
+ * shim's contract is "the encoder's piece sequence, nothing more"), plus exact hand-computed span
+ * assertions through every normalization step that changes, inserts, or removes characters.
+ */
+class WordpieceEncoderTest {
+
+ // Ids are indices: [PAD]=0, [UNK]=1, [CLS]=2, [SEP]=3, hello=4, world=5, ##s=6, won=7,
+ // ##der=8, ##ful=9, ca=10, ##fe=11, istanbul=12, U+4E2D=13, U+56FD=14, .=15, ,=16, !=17,
+ // he=18, ##llo=19, Greek "sofos" with a final sigma=20.
+ private static final List VOCAB = List.of(
+ "[PAD]", "[UNK]", "[CLS]", "[SEP]", "hello", "world", "##s", "won", "##der", "##ful",
+ "ca", "##fe", "istanbul", "\u4E2D", "\u56FD", ".", ",", "!", "he", "##llo",
+ "\u03C3\u03BF\u03C6\u03BF\u03C2");
+
+ private static WordpieceEncoder uncased() {
+ return new WordpieceEncoder(VOCAB);
+ }
+
+ private static void assertPiece(SubwordPiece piece, String expectedPiece, int expectedId,
+ int expectedStart, int expectedEnd) {
+ assertEquals(expectedPiece, piece.piece());
+ assertEquals(expectedId, piece.id());
+ assertEquals(expectedStart, piece.start(), "start of " + piece);
+ assertEquals(expectedEnd, piece.end(), "end of " + piece);
+ assertEquals(new Span(expectedStart, expectedEnd), piece.span(), "span of " + piece);
+ }
+
+ /**
+ * The curated parity inputs, each exercising a normalization step of the pipeline.
+ *
+ * @return The inputs.
+ */
+ static Stream curatedInputs() {
+ return Stream.of(
+ "",
+ " ",
+ "Hello, WORLD!",
+ "Wonderful",
+ "hellos",
+ // An accented e, stripped by NFD decomposition.
+ "Caf\u00E9",
+ // The Turkish dotted capital I: lower cases to two chars, then the dot strips away.
+ "\u0130stanbul",
+ // CJK ideographs are isolated into single-character tokens.
+ "\u4E2D\u56FD is CJK",
+ // Greek upper case: the trailing sigma takes the contextual final-sigma mapping.
+ "\u03A3\u039F\u03A6\u039F\u03A3",
+ // The NBSP is whitespace in the BERT sense.
+ "hello\u00A0world",
+ // NUL and the zero-width space are removed by the cleaning stage.
+ "a\u0000b\u200Bc",
+ // An emoji: unknown to the vocabulary, and a surrogate pair.
+ "\uD83D\uDE00",
+ "!!!",
+ "a".repeat(101),
+ "he said: \u00ABhello\u00BB.");
+ }
+
+ @ParameterizedTest
+ @MethodSource("curatedInputs")
+ @SuppressWarnings("removal") // BertTokenizer is pinned to the encoder until its removal in 3.1.
+ void testPieceSequenceMatchesBertTokenizerOnCuratedInputs(String input) {
+ final BertTokenizer bertTokenizer = new BertTokenizer(new HashSet<>(VOCAB), true);
+ final WordpieceEncoder encoder = uncased();
+ assertArrayEquals(bertTokenizer.tokenize(input), encoder.encodeToPieces(input),
+ "parity broke on: " + input);
+ }
+
+ @Test
+ @SuppressWarnings("removal") // BertTokenizer is pinned to the encoder until its removal in 3.1.
+ void testPieceSequenceMatchesBertTokenizerOnRandomInputs() {
+ final int[] pool = {'a', 'b', 'A', 'B', 'z', ' ', ' ', '\t', 0x00A0, '.', '!', ',',
+ 0x0301, 0x00E9, 0x0130, 0x03A3, 0x03C3, 0x03BF, 0x4E2D, 0xFFFD, 0x200B, 0x1F600, 0};
+ final Random random = new Random(42);
+ for (final boolean lowerCase : new boolean[] {true, false}) {
+ final BertTokenizer bertTokenizer = new BertTokenizer(new HashSet<>(VOCAB), lowerCase);
+ final WordpieceEncoder encoder = new WordpieceEncoder(VOCAB, lowerCase);
+ for (int round = 0; round < 400; round++) {
+ final StringBuilder text = new StringBuilder();
+ final int length = random.nextInt(25);
+ for (int i = 0; i < length; i++) {
+ text.appendCodePoint(pool[random.nextInt(pool.length)]);
+ }
+ final String input = text.toString();
+ assertArrayEquals(bertTokenizer.tokenize(input), encoder.encodeToPieces(input),
+ "parity broke on: " + input);
+
+ // Span invariants: within bounds and never moving backwards.
+ int previousStart = 0;
+ for (final SubwordPiece piece : encoder.encode(input)) {
+ assertTrue(piece.start() >= previousStart && piece.end() <= input.length(),
+ "span out of order or bounds in " + input + ": " + piece);
+ previousStart = piece.start();
+ }
+ }
+ }
+ }
+
+ @Test
+ void testSpansSurvivePunctuationIsolationAndCaseFolding() {
+ final List pieces = uncased().encode("Hello, WORLD!");
+ assertEquals(6, pieces.size());
+ assertPiece(pieces.get(0), "[CLS]", 2, 0, 0);
+ assertPiece(pieces.get(1), "hello", 4, 0, 5);
+ assertPiece(pieces.get(2), ",", 16, 5, 6);
+ assertPiece(pieces.get(3), "world", 5, 7, 12);
+ assertPiece(pieces.get(4), "!", 17, 12, 13);
+ assertPiece(pieces.get(5), "[SEP]", 3, 13, 13);
+ }
+
+ @Test
+ void testSpansSurviveAccentStripping() {
+ // The accent is stripped by NFD, yet ##fe still covers the accented surface.
+ final List pieces = uncased().encode("Caf\u00E9");
+ assertEquals(4, pieces.size());
+ assertPiece(pieces.get(1), "ca", 10, 0, 2);
+ assertPiece(pieces.get(2), "##fe", 11, 2, 4);
+ }
+
+ @Test
+ void testSpansSurviveLengthChangingLowerCasing() {
+ // The Turkish dotted capital I lower cases to two chars before the combining dot strips;
+ // the piece still covers the original eight chars.
+ final List pieces = uncased().encode("\u0130stanbul");
+ assertEquals(3, pieces.size());
+ assertPiece(pieces.get(1), "istanbul", 12, 0, 8);
+ }
+
+ @Test
+ void testCjkIsolationYieldsOnePieceAndSpanPerIdeograph() {
+ final List pieces = uncased().encode("\u4E2D\u56FD");
+ assertEquals(4, pieces.size());
+ assertPiece(pieces.get(1), "\u4E2D", 13, 0, 1);
+ assertPiece(pieces.get(2), "\u56FD", 14, 1, 2);
+ }
+
+ @Test
+ void testLineAndParagraphSeparatorsSplitWords() {
+ // Zl and Zp are not whitespace in the BERT _is_whitespace sense, but the reference
+ // pipeline's whitespace_tokenize (Python's str.split()) breaks words on them, as did the
+ // previous OpenNLP pipeline via WhitespaceTokenizer.
+ final List pieces = uncased().encode("hello\u2028world\u2029hello");
+ assertEquals(5, pieces.size());
+ assertPiece(pieces.get(1), "hello", 4, 0, 5);
+ assertPiece(pieces.get(2), "world", 5, 6, 11);
+ assertPiece(pieces.get(3), "hello", 4, 12, 17);
+ }
+
+ @Test
+ void testUnknownWordCoversItsWholeSurfaceIncludingRemovedChars() {
+ // NUL and the zero-width space are removed by cleaning, so one word "abc" remains; it is
+ // not representable and becomes the unknown piece spanning the full original surface.
+ final List pieces = uncased().encode("a\u0000b\u200Bc");
+ assertEquals(3, pieces.size());
+ assertPiece(pieces.get(1), "[UNK]", 1, 0, 5);
+ }
+
+ @Test
+ void testContextualCaseMappingFallsBackToWordWideSpans() {
+ // Greek final sigma is a contextual mapping the per-char rerun cannot reproduce, so the
+ // word's pieces fall back to spanning the whole word; the piece content is asserted
+ // exactly below.
+ final List pieces =
+ uncased().encode("\u03A3\u039F\u03A6\u039F\u03A3");
+ assertEquals(3, pieces.size());
+ assertPiece(pieces.get(1),
+ "\u03C3\u03BF\u03C6\u03BF\u03C2", 20, 0, 5);
+ }
+
+ @Test
+ void testEncodeToIdsCarriesVocabularyLineNumbers() {
+ assertArrayEquals(new int[] {2, 4, 5, 6, 3}, uncased().encodeToIds("Hello worldS"));
+ }
+
+ @Test
+ void testCasedEncoderKeepsCase() {
+ final List vocabulary = new ArrayList<>(VOCAB);
+ vocabulary.add("Hello");
+ final WordpieceEncoder cased = new WordpieceEncoder(vocabulary, false);
+ final List pieces = cased.encode("Hello hello");
+ assertPiece(pieces.get(1), "Hello", vocabulary.size() - 1, 0, 5);
+ assertPiece(pieces.get(2), "hello", 4, 6, 11);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", " "})
+ void testEmptyAndBlankTextEncodeToTheFramePiecesOnly(String input) {
+ final List pieces = uncased().encode(input);
+ assertEquals(2, pieces.size(), "frame pieces broke on <" + input + ">");
+ assertPiece(pieces.get(0), "[CLS]", 2, 0, 0);
+ assertPiece(pieces.get(1), "[SEP]", 3, input.length(), input.length());
+ }
+
+ @Test
+ void testValidationFailsLoudly() {
+ assertThrows(IllegalArgumentException.class, () -> new WordpieceEncoder(null));
+ assertThrows(IllegalArgumentException.class,
+ () -> new WordpieceEncoder(List.of("[CLS]", "[SEP]")));
+ assertThrows(IllegalArgumentException.class,
+ () -> new WordpieceEncoder(List.of("[CLS]", "[SEP]", "[UNK]", "dup", "dup")));
+ final List withNull = new ArrayList<>(VOCAB);
+ withNull.add(null);
+ assertThrows(IllegalArgumentException.class, () -> new WordpieceEncoder(withNull));
+ assertThrows(IllegalArgumentException.class, () -> uncased().encode(null));
+ }
+}
diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml
index cd1d8a2ddf..c3906cd144 100644
--- a/opennlp-docs/src/docbkx/tokenizer.xml
+++ b/opennlp-docs/src/docbkx/tokenizer.xml
@@ -539,4 +539,85 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> {
+
+ Subword Tokenization
+
+ Neural models usually operate on subword units drawn from a fixed vocabulary rather than
+ on words. The SubwordTokenizer interface in
+ opennlp.tools.tokenize covers this case: encode splits text into
+ pieces, and every returned SubwordPiece carries the piece string in the
+ model's normalized form, its vocabulary id, and the exact span of the original text it
+ came from. Because the segmentation runs after the model's own normalization, a piece is
+ generally not a substring of the input; the spans always refer to the caller's original
+ text, so annotations computed over the pieces can be mapped back without guesswork. The
+ encodeToIds and encodeToPieces methods return just the ids or
+ the piece strings when the spans are not needed. Thread safety is implementation specific;
+ both implementations described below are immutable and safe for concurrent use by
+ multiple threads.
+
+
+ SentencePiece
+
+ SentencePieceTokenizer in the opennlp-subword artifact
+ (package opennlp.subword.sentencepiece) runs a trained SentencePiece
+ .model file purely in Java, with no native library. The file itself
+ carries everything the class needs: the vocabulary with piece scores and types, the
+ segmentation algorithm (unigram language model or byte-pair encoding), and the text
+ normalizer the model was trained with. Loading is a one-time cost and the resulting
+ instance is immutable, so one tokenizer can be shared by any number of threads.
+
+ id " + piece.id()
+ + ", original text [" + piece.start() + ", " + piece.end() + ")");
+}
+
+int[] ids = tokenizer.encodeToIds("Ready for the embedding layer.");]]>
+
+ SentencePieceUsageExampleTest asserts the load-and-encode workflow
+ shown here. The vocabulary can be inspected through
+ vocabularySize, idToPiece, pieceToId, and
+ score, and the algorithm method reports whether the
+ model uses the unigram or the byte-pair encoding algorithm.
+
+
+ The model's own normalizer is also exposed directly:
+ SentencePieceTokenizer implements OffsetAwareNormalizer, so
+ normalize applies the model's normalization rules to arbitrary text and
+ normalizeAligned additionally returns the character alignment described
+ in . This is useful when other processing must see
+ text exactly as the subword model does.
+
+
+
+ WordPiece
+
+ WordpieceEncoder in opennlp.tools.tokenize runs the full
+ BERT tokenization pipeline: control character removal, whitespace normalization, CJK
+ isolation, optional lower casing with accent stripping, punctuation isolation, and
+ greedy longest-match wordpiece segmentation. It is constructed from the model's
+ vocabulary, conventionally the lines of a vocab.txt file, where a piece's
+ id is its line number.
+
+ vocabulary = Files.readAllLines(Path.of("vocab.txt"));
+
+WordpieceEncoder encoder = new WordpieceEncoder(vocabulary); // uncased model
+WordpieceEncoder cased = new WordpieceEncoder(vocabulary, false); // cased model
+
+List pieces = encoder.encode("OpenNLP encodes text for BERT models.");
+int[] ids = encoder.encodeToIds("OpenNLP encodes text for BERT models.");]]>
+
+ Every encoding is framed by the classification and separator pieces, which carry empty
+ spans at the text's boundaries, and words the vocabulary cannot cover become the
+ unknown piece. Models with other special tokens or with non-contiguous ids are
+ supported through the constructors taking explicit special tokens or a piece-to-id
+ map. The older BertTokenizer is deprecated and scheduled for removal:
+ its tokenize method returns the same piece sequence that
+ encodeToPieces returns. Migrating means passing the vocabulary as a
+ List or a piece-to-id map instead of a Set; the output
+ is unchanged.
+
+
+
diff --git a/opennlp-extensions/opennlp-subword/pom.xml b/opennlp-extensions/opennlp-subword/pom.xml
new file mode 100644
index 0000000000..b4d13c755e
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/pom.xml
@@ -0,0 +1,58 @@
+
+
+
+
+
+ 4.0.0
+
+ org.apache.opennlp
+ opennlp-extensions
+ 3.0.0-SNAPSHOT
+
+
+ opennlp-subword
+ jar
+ Apache OpenNLP :: Ext :: Subword
+
+
+
+ org.apache.opennlp
+ opennlp-api
+
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ test
+
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ test
+
+
+
+ org.junit.jupiter
+ junit-jupiter-params
+ test
+
+
+
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java
new file mode 100644
index 0000000000..6e448e5c38
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+
+/**
+ * Byte-pair-encoding segmentation: the normalized text starts as single characters (or
+ * user-defined symbols, which are frozen whole) and adjacent pairs merge greedily, highest piece
+ * score first, until no adjacent pair forms a vocabulary piece.
+ *
+ * Only pieces of the normal, user-defined, and unused types participate in merges; a merge that
+ * lands on an unused piece is re-segmented back into its constituents.
+ */
+final class BpeEncoder implements Serializable {
+
+ private static final long serialVersionUID = -57799941356582785L;
+
+ private static final int MAX_RESEGMENT_DEPTH = 100;
+
+ private final Map pieces;
+ private final float[] scores;
+ private final boolean[] unused;
+ private final boolean[] reserved;
+ private final int unkId;
+ private final PieceTrie userDefinedMatcher;
+
+ /**
+ * Instantiates the encoder.
+ *
+ * @param pieces All pieces by content, mapping to their ids.
+ * @param scores The score of every piece, indexed by id.
+ * @param unused Whether each id has the unused piece type.
+ * @param reserved Whether each id is excluded from merging (any type other than
+ * normal, user-defined, or unused).
+ * @param unkId The id of the unknown piece.
+ * @param userDefinedMatcher Longest-match trie over user-defined symbols, or null when the
+ * model defines none.
+ */
+ BpeEncoder(Map pieces, float[] scores, boolean[] unused, boolean[] reserved,
+ int unkId, PieceTrie userDefinedMatcher) {
+ this.pieces = pieces;
+ this.scores = scores;
+ this.unused = unused;
+ this.reserved = reserved;
+ this.unkId = unkId;
+ this.userDefinedMatcher = userDefinedMatcher;
+ }
+
+ /**
+ * A candidate merge of the symbols at indices {@code left} and {@code right}; {@code size} is the
+ * merged byte length, used to detect staleness after either side has changed.
+ */
+ private record Pair(int left, int right, float score, int size) {
+ }
+
+ /**
+ * Segments normalized text.
+ *
+ * @param normalized The buffer holding the normalized UTF-8 bytes; must not be null.
+ * @param size The number of valid bytes in {@code normalized}.
+ * @return The segments covering all bytes, in text order.
+ * @throws IllegalArgumentException Thrown if {@code normalized} is null.
+ */
+ List encode(byte[] normalized, int size) {
+ if (normalized == null) {
+ throw new IllegalArgumentException("normalized must not be null");
+ }
+ if (size == 0) {
+ return List.of();
+ }
+
+ // The symbol list as index-linked ranges of the normalized bytes; merged-away symbols
+ // become empty ranges.
+ final IntBuilder fromB = new IntBuilder(size);
+ final IntBuilder toB = new IntBuilder(size);
+ final List freezeList = new ArrayList<>();
+ int position = 0;
+ while (position < size) {
+ int matched = 0;
+ if (userDefinedMatcher != null) {
+ matched = userDefinedMatcher.longestMatch(normalized, size, position);
+ }
+ final boolean frozen = matched > 0;
+ final int length = frozen ? matched
+ : Math.min(SentencePieceNormalizer.utf8Length(normalized[position]),
+ size - position);
+ fromB.append(position);
+ toB.append(position + length);
+ freezeList.add(frozen);
+ position += length;
+ }
+ final int symbolCount = freezeList.size();
+ final int[] from = fromB.toArray();
+ final int[] to = toB.toArray();
+ final int[] prev = new int[symbolCount];
+ final int[] next = new int[symbolCount];
+ final boolean[] freeze = new boolean[symbolCount];
+ for (int i = 0; i < symbolCount; i++) {
+ prev[i] = i - 1;
+ next[i] = i + 1 < symbolCount ? i + 1 : -1;
+ freeze[i] = freezeList.get(i);
+ }
+
+ // Higher score first; equal scores break towards the leftmost pair.
+ final PriorityQueue agenda = new PriorityQueue<>((a, b) -> {
+ final int byScore = Float.compare(b.score(), a.score());
+ return byScore != 0 ? byScore : Integer.compare(a.left(), b.left());
+ });
+ // Merged piece content mapped back to its two constituents, for re-segmenting unused pieces.
+ final Map revMerge = new HashMap<>();
+
+ for (int left = 0; left + 1 < symbolCount; left++) {
+ maybeAddPair(normalized, from, to, freeze, left, left + 1, agenda, revMerge);
+ }
+
+ while (!agenda.isEmpty()) {
+ final Pair top = agenda.poll();
+ // Skips entries made stale by an earlier merge of either side.
+ if (from[top.left()] == to[top.left()] || from[top.right()] == to[top.right()]
+ || to[top.left()] - from[top.left()] + to[top.right()] - from[top.right()]
+ != top.size()) {
+ continue;
+ }
+
+ // Replaces the pair with the merged symbol.
+ to[top.left()] = to[top.right()];
+ next[top.left()] = next[top.right()];
+ if (next[top.right()] >= 0) {
+ prev[next[top.right()]] = top.left();
+ }
+ from[top.right()] = to[top.right()];
+
+ maybeAddPair(normalized, from, to, freeze, prev[top.left()], top.left(), agenda, revMerge);
+ maybeAddPair(normalized, from, to, freeze, top.left(), next[top.left()], agenda, revMerge);
+ }
+
+ final List output = new ArrayList<>(symbolCount);
+ int consumed = 0;
+ for (int index = 0; index != -1; index = next[index]) {
+ final String piece =
+ new String(normalized, from[index], to[index] - from[index], StandardCharsets.UTF_8);
+ consumed = resegment(piece, consumed, 0, revMerge, output);
+ }
+ return output;
+ }
+
+ /**
+ * Offers the adjacent symbol pair {@code (left, right)} as a merge candidate: the pair joins
+ * the agenda only when the concatenation is a mergeable vocabulary piece, and a merge landing
+ * on an unused piece is remembered in {@code revMerge} for later re-segmentation.
+ *
+ * @param normalized The buffer holding the normalized UTF-8 bytes.
+ * @param from Per symbol, the inclusive start offset in {@code normalized}.
+ * @param to Per symbol, the exclusive end offset in {@code normalized}.
+ * @param freeze Per symbol, whether it is a user-defined symbol excluded from merging.
+ * @param left The index of the left symbol, or {@code -1} for none.
+ * @param right The index of the right symbol, or {@code -1} for none.
+ * @param agenda The merge agenda to add to.
+ * @param revMerge The map from a merged piece to its two constituents.
+ */
+ private void maybeAddPair(byte[] normalized, int[] from, int[] to, boolean[] freeze,
+ int left, int right, PriorityQueue agenda,
+ Map revMerge) {
+ if (left == -1 || right == -1 || freeze[left] || freeze[right]) {
+ return;
+ }
+ final String piece =
+ new String(normalized, from[left], to[right] - from[left], StandardCharsets.UTF_8);
+ final Integer id = pieces.get(piece);
+ if (id == null || id == unkId || reserved[id]) {
+ return;
+ }
+ agenda.add(new Pair(left, right, scores[id], to[right] - from[left]));
+ if (unused[id]) {
+ revMerge.put(piece, new String[] {
+ new String(normalized, from[left], to[left] - from[left], StandardCharsets.UTF_8),
+ new String(normalized, from[right], to[right] - from[right], StandardCharsets.UTF_8)});
+ }
+ }
+
+ /**
+ * Emits a symbol, splitting a piece of the unused type back into the pieces it was merged from.
+ * Positions are assigned by a running cursor; constituent byte lengths always sum to the merged
+ * length, so the cursor stays aligned with the normalized bytes.
+ *
+ * @param piece The piece content to emit.
+ * @param consumed The running byte cursor into the normalized text.
+ * @param depth The current recursion depth.
+ * @param revMerge The map from a merged piece to its two constituents.
+ * @param output The segment list to append to.
+ * @return The updated byte cursor.
+ */
+ private int resegment(String piece, int consumed, int depth, Map revMerge,
+ List output) {
+ final Integer mapped = pieces.get(piece);
+ final int id = mapped == null ? unkId : mapped;
+ final int byteLength = piece.getBytes(StandardCharsets.UTF_8).length;
+ if (depth > MAX_RESEGMENT_DEPTH || !unused[id]) {
+ output.add(new Segment(consumed, consumed + byteLength, id));
+ return consumed + byteLength;
+ }
+ final String[] parts = revMerge.get(piece);
+ if (parts == null) {
+ output.add(new Segment(consumed, consumed + byteLength, id));
+ return consumed + byteLength;
+ }
+ consumed = resegment(parts[0], consumed, depth + 1, revMerge, output);
+ return resegment(parts[1], consumed, depth + 1, revMerge, output);
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java
new file mode 100644
index 0000000000..432f2104dc
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.util.Arrays;
+
+/** A growable byte buffer supporting append, truncate, and suffix comparison. */
+final class ByteBuilder {
+
+ /** The smallest backing array, so tiny requested capacities still grow geometrically. */
+ private static final int MIN_CAPACITY = 16;
+
+ private byte[] data;
+ private int length;
+
+ /**
+ * Instantiates the buffer.
+ *
+ * @param capacity The initial capacity hint.
+ */
+ ByteBuilder(int capacity) {
+ data = new byte[Math.max(capacity, MIN_CAPACITY)];
+ }
+
+ /**
+ * Appends one byte.
+ *
+ * @param b The byte to append.
+ */
+ void append(byte b) {
+ if (length == data.length) {
+ data = Arrays.copyOf(data, grownLength());
+ }
+ data[length++] = b;
+ }
+
+ /**
+ * Appends a run of bytes.
+ *
+ * @param source The source array.
+ * @param from The inclusive start offset in {@code source}.
+ * @param count The number of bytes to append.
+ */
+ void append(byte[] source, int from, int count) {
+ while (length + count > data.length) {
+ data = Arrays.copyOf(data, grownLength());
+ }
+ System.arraycopy(source, from, data, length, count);
+ length += count;
+ }
+
+ /** {@return the next backing-array length under the 1.5x growth policy} */
+ private int grownLength() {
+ return data.length + (data.length >> 1);
+ }
+
+ /** {@return the number of valid bytes} */
+ int length() {
+ return length;
+ }
+
+ /**
+ * Shrinks the valid length.
+ *
+ * @param newLength The new length, not negative and not greater than the current length.
+ * @throws IllegalArgumentException Thrown if {@code newLength} is negative or greater than the
+ * current length.
+ */
+ void truncate(int newLength) {
+ if (newLength < 0 || newLength > length) {
+ throw new IllegalArgumentException(
+ "The new length " + newLength + " is outside [0, " + length + "].");
+ }
+ length = newLength;
+ }
+
+ /**
+ * Tests whether the valid bytes end with the given suffix.
+ *
+ * @param suffix The suffix to test.
+ * @return {@code true} when the buffer ends with {@code suffix}.
+ */
+ boolean endsWith(byte[] suffix) {
+ if (length < suffix.length) {
+ return false;
+ }
+ return Arrays.equals(data, length - suffix.length, length, suffix, 0, suffix.length);
+ }
+
+ /** {@return a trimmed copy of the valid bytes} */
+ byte[] toArray() {
+ return Arrays.copyOf(data, length);
+ }
+
+ /** {@return the backing array, valid up to {@link #length()}} */
+ byte[] array() {
+ return data;
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java
new file mode 100644
index 0000000000..75f48e1de8
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.Serializable;
+
+/**
+ * Read-only lookup over a serialized Darts-clone double-array trie, the dictionary format
+ * embedded in a SentencePiece model's precompiled character map.
+ *
+ * Each unit is one little-endian 32-bit word encoding a label, an offset to the unit's
+ * children, and a leaf flag; traversal XORs the offset with the next key byte. Only the longest
+ * prefix match is needed here, so this walks the byte key once and remembers the last accepting
+ * state. Out-of-range unit references, which a well-formed trie never produces, fail loudly
+ * rather than reading arbitrary memory.
+ *
+ * @see Darts-clone
+ */
+final class DoubleArrayTrie implements Serializable {
+
+ private static final long serialVersionUID = -1572336116472261588L;
+
+ // A non-leaf unit stores its transition label in the low 8 bits and the leaf flag in the sign
+ // bit. Key bytes are in [0, 255] with the sign bit clear, so comparing (unit & this mask)
+ // against a key byte both matches the label and rejects leaf units in one test.
+ private static final int LEAF_FLAG_AND_LABEL_MASK = 0x800000FF;
+
+ // A leaf unit stores the key's value in its low 31 bits; the sign bit is the leaf flag.
+ private static final int LEAF_VALUE_MASK = 0x7FFFFFFF;
+
+ // Bit 8 of a non-leaf unit marks that one of its children is a leaf holding this key's value.
+ private static final int HAS_LEAF_BIT = 8;
+
+ private final int[] units;
+
+ /**
+ * Wraps serialized trie units.
+ *
+ * @param data The bytes holding the units; must not be null.
+ * @param offset The offset of the first unit byte.
+ * @param length The number of bytes; must be a positive multiple of four.
+ * @throws IllegalArgumentException Thrown if {@code length} is not a positive multiple of four.
+ */
+ DoubleArrayTrie(byte[] data, int offset, int length) {
+ if (length <= 0 || (length & 3) != 0) {
+ throw new IllegalArgumentException(
+ "The trie length " + length + " is not a positive multiple of four bytes.");
+ }
+ units = new int[length >> 2];
+ for (int i = 0; i < units.length; i++) {
+ final int base = offset + (i << 2);
+ units[i] = (data[base] & 0xFF) | (data[base + 1] & 0xFF) << 8
+ | (data[base + 2] & 0xFF) << 16 | (data[base + 3] & 0xFF) << 24;
+ }
+ }
+
+ /**
+ * Finds the longest key that is a prefix of {@code key[from, to)}.
+ *
+ * @param key The byte key to match against; must not be null.
+ * @param from The inclusive start of the query window.
+ * @param to The exclusive end of the query window.
+ * @return {@code (value << 32) | matchedLength} for the longest match, or {@code -1} when no
+ * key matches. Values are non-negative, so the result is negative only on no-match.
+ * @throws IllegalArgumentException Thrown if the trie data references a unit outside its
+ * bounds, indicating corrupt data.
+ */
+ long longestPrefixMatch(byte[] key, int from, int to) {
+ // The JVM's own bounds checks guard the walk; the catch below translates an out-of-range unit
+ // reference from corrupt data into a loud failure.
+ final int[] u = units;
+ try {
+ long result = -1;
+ int nodePos = 0;
+ int unit = u[0];
+ nodePos ^= offset(unit);
+ for (int i = from; i < to; i++) {
+ final int b = key[i] & 0xFF;
+ nodePos ^= b;
+ unit = u[nodePos];
+ if ((unit & LEAF_FLAG_AND_LABEL_MASK) != b) {
+ return result;
+ }
+ nodePos ^= offset(unit);
+ if (((unit >>> HAS_LEAF_BIT) & 1) == 1) {
+ final int value = u[nodePos] & LEAF_VALUE_MASK;
+ result = ((long) value << 32) | (i - from + 1);
+ }
+ }
+ return result;
+ } catch (ArrayIndexOutOfBoundsException e) {
+ throw new IllegalArgumentException(
+ "The trie references a unit outside its " + u.length + " units.", e);
+ }
+ }
+
+ /**
+ * Tests whether any key starts with the given byte, which is exactly whether the root has a
+ * transition on it; used to precompute the first-byte gate of the normalizer scan.
+ *
+ * @param b The first key byte as an unsigned value.
+ * @return {@code true} if some key starts with {@code b}.
+ */
+ boolean hasTransitionFromRoot(int b) {
+ final int root = units[0];
+ final int nodePos = offset(root) ^ b;
+ if (nodePos < 0 || nodePos >= units.length) {
+ return false;
+ }
+ return (units[nodePos] & LEAF_FLAG_AND_LABEL_MASK) == b;
+ }
+
+ /**
+ * Returns the offset from a unit to its children, as encoded by Darts-clone: bits 10 to 30 hold
+ * the raw offset, and bit 9 is an extension flag that scales it by 256 for far-away children.
+ * The expression {@code (unit & (1 << 9)) >>> 6} evaluates to 8 exactly when bit 9 is set, so
+ * the raw offset is shifted left by either 0 or 8 bits.
+ *
+ * @param unit The unit word.
+ * @return The child offset.
+ */
+ private static int offset(int unit) {
+ return (unit >>> 10) << ((unit & (1 << 9)) >>> 6);
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java
new file mode 100644
index 0000000000..b06fb73209
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.util.Arrays;
+
+/** A growable int buffer supporting append, indexed read, and truncate. */
+final class IntBuilder {
+
+ /** The smallest backing array, so tiny requested capacities still grow geometrically. */
+ private static final int MIN_CAPACITY = 16;
+
+ private int[] data;
+ private int length;
+
+ /**
+ * Instantiates the buffer.
+ *
+ * @param capacity The initial capacity hint.
+ */
+ IntBuilder(int capacity) {
+ data = new int[Math.max(capacity, MIN_CAPACITY)];
+ }
+
+ /**
+ * Appends one value.
+ *
+ * @param value The value to append.
+ */
+ void append(int value) {
+ if (length == data.length) {
+ data = Arrays.copyOf(data, grownLength());
+ }
+ data[length++] = value;
+ }
+
+ /** {@return the next backing-array length under the 1.5x growth policy} */
+ private int grownLength() {
+ return data.length + (data.length >> 1);
+ }
+
+ /**
+ * Reads a value by index.
+ *
+ * @param index An index in {@code [0, length())}.
+ * @return The value at {@code index}.
+ * @throws IndexOutOfBoundsException Thrown if {@code index} is out of range.
+ */
+ int get(int index) {
+ if (index >= length) {
+ throw new IndexOutOfBoundsException("index " + index + " is outside [0, " + length + ")");
+ }
+ return data[index];
+ }
+
+ /** {@return the number of valid values} */
+ int length() {
+ return length;
+ }
+
+ /**
+ * Shrinks the valid length.
+ *
+ * @param newLength The new length, not negative and not greater than the current length.
+ * @throws IllegalArgumentException Thrown if {@code newLength} is negative or greater than the
+ * current length.
+ */
+ void truncate(int newLength) {
+ if (newLength < 0 || newLength > length) {
+ throw new IllegalArgumentException(
+ "The new length " + newLength + " is outside [0, " + length + "].");
+ }
+ length = newLength;
+ }
+
+ /** {@return a trimmed copy of the valid values} */
+ int[] toArray() {
+ return Arrays.copyOf(data, length);
+ }
+
+ /** {@return the backing array, valid up to {@link #length()}} */
+ int[] array() {
+ return data;
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java
new file mode 100644
index 0000000000..a6ceca6b84
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java
@@ -0,0 +1,419 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import opennlp.tools.util.InvalidFormatException;
+
+/**
+ * Reads the binary {@code ModelProto} serialization of a SentencePiece {@code .model} file.
+ *
+ * The format is the standard protocol-buffer wire encoding of one flat message
+ * ({@code sentencepiece_model.proto}, Apache License 2.0). This reader walks the tag stream
+ * directly and keeps only the fields inference needs: the pieces with scores and types, the
+ * normalizer spec, the trainer-spec fields that change runtime behavior, and the embedded
+ * self-test samples. Unknown fields are skipped, and malformed input fails loudly.
+ *
+ * @see
+ * sentencepiece_model.proto
+ */
+final class ModelProtoReader {
+
+ // Wire types of the protocol-buffer encoding.
+ private static final int WIRE_VARINT = 0;
+ private static final int WIRE_FIXED64 = 1;
+ private static final int WIRE_LEN = 2;
+ private static final int WIRE_FIXED32 = 5;
+
+ // Field numbers of the ModelProto message in sentencepiece_model.proto.
+ private static final int FIELD_MODEL_PIECES = 1;
+ private static final int FIELD_MODEL_TRAINER_SPEC = 2;
+ private static final int FIELD_MODEL_NORMALIZER_SPEC = 3;
+ private static final int FIELD_MODEL_SELF_TEST_DATA = 4;
+
+ // Field numbers of the ModelProto.SentencePiece sub-message.
+ private static final int FIELD_PIECE_PIECE = 1;
+ private static final int FIELD_PIECE_SCORE = 2;
+ private static final int FIELD_PIECE_TYPE = 3;
+
+ // Field numbers of the TrainerSpec sub-message.
+ private static final int FIELD_TRAINER_MODEL_TYPE = 3;
+ private static final int FIELD_TRAINER_TREAT_WHITESPACE_AS_SUFFIX = 24;
+ private static final int FIELD_TRAINER_BYTE_FALLBACK = 35;
+ private static final int FIELD_TRAINER_UNK_ID = 40;
+
+ // Field numbers of the NormalizerSpec sub-message.
+ private static final int FIELD_NORMALIZER_PRECOMPILED_CHARSMAP = 2;
+ private static final int FIELD_NORMALIZER_ADD_DUMMY_PREFIX = 3;
+ private static final int FIELD_NORMALIZER_REMOVE_EXTRA_WHITESPACES = 4;
+ private static final int FIELD_NORMALIZER_ESCAPE_WHITESPACES = 5;
+
+ // Field numbers of the SelfTestData sub-message and its Sample entries.
+ private static final int FIELD_SELF_TEST_SAMPLES = 1;
+ private static final int FIELD_SAMPLE_INPUT = 1;
+ private static final int FIELD_SAMPLE_EXPECTED = 2;
+
+ private final byte[] data;
+ private int pos;
+
+ /**
+ * Prepares a reader positioned at the start of the given bytes; {@link #read(byte[])} drives
+ * the actual parse.
+ *
+ * @param data The raw bytes of a {@code .model} file.
+ */
+ private ModelProtoReader(byte[] data) {
+ this.data = data;
+ }
+
+ /**
+ * Parses a serialized {@code ModelProto}.
+ *
+ * @param data The raw bytes of a {@code .model} file; must not be null.
+ * @return The parsed model description.
+ * @throws IllegalArgumentException Thrown if {@code data} is null.
+ * @throws InvalidFormatException Thrown if the bytes are not a well-formed model.
+ */
+ static RawModel read(byte[] data) throws InvalidFormatException {
+ if (data == null) {
+ throw new IllegalArgumentException("data must not be null");
+ }
+ final ModelProtoReader reader = new ModelProtoReader(data);
+ final RawModel model = new RawModel();
+ while (reader.pos < data.length) {
+ final long tag = reader.varint();
+ switch (fieldOf(tag)) {
+ case FIELD_MODEL_PIECES -> reader.piece(model, reader.lenPayload(tag));
+ case FIELD_MODEL_TRAINER_SPEC -> reader.trainerSpec(model, reader.lenPayload(tag));
+ case FIELD_MODEL_NORMALIZER_SPEC -> reader.normalizerSpec(model, reader.lenPayload(tag));
+ case FIELD_MODEL_SELF_TEST_DATA -> reader.selfTestData(model, reader.lenPayload(tag));
+ default -> reader.skip(tag);
+ }
+ }
+ if (model.pieces.isEmpty()) {
+ throw new InvalidFormatException("The model defines no pieces.");
+ }
+ return model;
+ }
+
+ /**
+ * Extracts the field number from a wire-format tag.
+ *
+ * @param tag The field tag.
+ * @return The field number.
+ */
+ private static int fieldOf(long tag) {
+ return (int) (tag >>> 3);
+ }
+
+ /**
+ * Extracts the wire type from a wire-format tag.
+ *
+ * @param tag The field tag.
+ * @return The wire type.
+ */
+ private static int wireTypeOf(long tag) {
+ return (int) (tag & 7);
+ }
+
+ /**
+ * Parses one {@code SentencePiece} sub-message and appends its piece, score, and type.
+ *
+ * @param model The model to append to.
+ * @param end The exclusive end offset of the sub-message payload.
+ * @throws InvalidFormatException Thrown if the sub-message is malformed or defines an empty
+ * piece or a non-finite score.
+ */
+ private void piece(RawModel model, int end) throws InvalidFormatException {
+ String piece = null;
+ float score = 0;
+ int type = RawModel.TYPE_NORMAL;
+ while (pos < end) {
+ final long tag = varint();
+ switch (fieldOf(tag)) {
+ case FIELD_PIECE_PIECE -> piece = utf8(lenPayload(tag));
+ case FIELD_PIECE_SCORE -> score = fixed32Float(tag);
+ case FIELD_PIECE_TYPE -> type = (int) varintOf(tag);
+ default -> skip(tag);
+ }
+ }
+ if (piece == null || piece.isEmpty()) {
+ throw new InvalidFormatException(
+ "The model contains an empty piece at index " + model.pieces.size() + ".");
+ }
+ if (Float.isNaN(score) || Float.isInfinite(score)) {
+ throw new InvalidFormatException("The score of piece '" + piece + "' is not finite.");
+ }
+ model.pieces.add(piece);
+ model.scores.add(score);
+ model.types.add(type);
+ }
+
+ /**
+ * Parses the {@code TrainerSpec} sub-message, keeping the fields that change runtime behavior.
+ *
+ * @param model The model to populate.
+ * @param end The exclusive end offset of the sub-message payload.
+ * @throws InvalidFormatException Thrown if the sub-message is malformed.
+ */
+ private void trainerSpec(RawModel model, int end) throws InvalidFormatException {
+ while (pos < end) {
+ final long tag = varint();
+ switch (fieldOf(tag)) {
+ case FIELD_TRAINER_MODEL_TYPE -> model.modelType = (int) varintOf(tag);
+ case FIELD_TRAINER_TREAT_WHITESPACE_AS_SUFFIX ->
+ model.treatWhitespaceAsSuffix = varintOf(tag) != 0;
+ case FIELD_TRAINER_BYTE_FALLBACK -> model.byteFallback = varintOf(tag) != 0;
+ case FIELD_TRAINER_UNK_ID -> model.unkId = (int) varintOf(tag);
+ default -> skip(tag);
+ }
+ }
+ }
+
+ /**
+ * Parses the {@code NormalizerSpec} sub-message: the precompiled character map and the
+ * whitespace-handling flags.
+ *
+ * @param model The model to populate.
+ * @param end The exclusive end offset of the sub-message payload.
+ * @throws InvalidFormatException Thrown if the sub-message is malformed.
+ */
+ private void normalizerSpec(RawModel model, int end) throws InvalidFormatException {
+ while (pos < end) {
+ final long tag = varint();
+ switch (fieldOf(tag)) {
+ case FIELD_NORMALIZER_PRECOMPILED_CHARSMAP ->
+ model.precompiledCharsMap = bytes(lenPayload(tag));
+ case FIELD_NORMALIZER_ADD_DUMMY_PREFIX -> model.addDummyPrefix = varintOf(tag) != 0;
+ case FIELD_NORMALIZER_REMOVE_EXTRA_WHITESPACES ->
+ model.removeExtraWhitespaces = varintOf(tag) != 0;
+ case FIELD_NORMALIZER_ESCAPE_WHITESPACES -> model.escapeWhitespaces = varintOf(tag) != 0;
+ default -> skip(tag);
+ }
+ }
+ }
+
+ /**
+ * Parses the {@code SelfTestData} sub-message, collecting the input and expected-segmentation
+ * sample pairs.
+ *
+ * @param model The model to populate.
+ * @param end The exclusive end offset of the sub-message payload.
+ * @throws InvalidFormatException Thrown if the sub-message is malformed.
+ */
+ private void selfTestData(RawModel model, int end) throws InvalidFormatException {
+ while (pos < end) {
+ final long tag = varint();
+ if (fieldOf(tag) == FIELD_SELF_TEST_SAMPLES) {
+ final int sampleEnd = lenPayload(tag);
+ String input = null;
+ String expected = null;
+ while (pos < sampleEnd) {
+ final long sampleTag = varint();
+ switch (fieldOf(sampleTag)) {
+ case FIELD_SAMPLE_INPUT -> input = utf8(lenPayload(sampleTag));
+ case FIELD_SAMPLE_EXPECTED -> expected = utf8(lenPayload(sampleTag));
+ default -> skip(sampleTag);
+ }
+ }
+ if (input != null && expected != null) {
+ model.selfTestInputs.add(input);
+ model.selfTestExpected.add(expected);
+ }
+ } else {
+ skip(tag);
+ }
+ }
+ }
+
+ /**
+ * Reads the length prefix of a length-delimited field and returns the exclusive end offset of its
+ * payload.
+ *
+ * @param tag The field tag, whose wire type must be length-delimited.
+ * @return The exclusive end offset of the payload.
+ * @throws InvalidFormatException Thrown if the wire type is wrong or the length runs past the
+ * input.
+ */
+ private int lenPayload(long tag) throws InvalidFormatException {
+ if (wireTypeOf(tag) != WIRE_LEN) {
+ throw malformed("field " + fieldOf(tag) + " is not length-delimited");
+ }
+ final long length = varint();
+ if (length < 0 || pos + length > data.length) {
+ throw malformed("length " + length + " exceeds the remaining input");
+ }
+ return pos + (int) length;
+ }
+
+ /**
+ * Reads the varint value of a field after checking its wire type.
+ *
+ * @param tag The field tag, whose wire type must be varint.
+ * @return The decoded value.
+ * @throws InvalidFormatException Thrown if the wire type is wrong or the varint is malformed.
+ */
+ private long varintOf(long tag) throws InvalidFormatException {
+ if (wireTypeOf(tag) != WIRE_VARINT) {
+ throw malformed("field " + fieldOf(tag) + " is not a varint");
+ }
+ return varint();
+ }
+
+ /**
+ * Reads the little-endian 32-bit float value of a field after checking its wire type.
+ *
+ * @param tag The field tag, whose wire type must be 32-bit.
+ * @return The decoded float.
+ * @throws InvalidFormatException Thrown if the wire type is wrong or the input is truncated.
+ */
+ private float fixed32Float(long tag) throws InvalidFormatException {
+ if (wireTypeOf(tag) != WIRE_FIXED32) {
+ throw malformed("field " + fieldOf(tag) + " is not a 32-bit value");
+ }
+ if (pos + 4 > data.length) {
+ throw malformed("truncated 32-bit value");
+ }
+ final int bits = (data[pos] & 0xFF) | (data[pos + 1] & 0xFF) << 8
+ | (data[pos + 2] & 0xFF) << 16 | (data[pos + 3] & 0xFF) << 24;
+ pos += 4;
+ return Float.intBitsToFloat(bits);
+ }
+
+ /**
+ * Decodes the bytes from the current position up to {@code end} as UTF-8, advancing past them.
+ *
+ * @param end The exclusive end offset of the payload.
+ * @return The decoded string.
+ */
+ private String utf8(int end) {
+ final String s = new String(data, pos, end - pos, StandardCharsets.UTF_8);
+ pos = end;
+ return s;
+ }
+
+ /**
+ * Copies the bytes from the current position up to {@code end}, advancing past them.
+ *
+ * @param end The exclusive end offset of the payload.
+ * @return The copied bytes.
+ */
+ private byte[] bytes(int end) {
+ final byte[] b = new byte[end - pos];
+ System.arraycopy(data, pos, b, 0, b.length);
+ pos = end;
+ return b;
+ }
+
+ /**
+ * Reads a base-128 varint from the current position, advancing past it.
+ *
+ * @return The decoded value.
+ * @throws InvalidFormatException Thrown if the input ends mid-varint or the varint exceeds 64
+ * bits.
+ */
+ private long varint() throws InvalidFormatException {
+ long value = 0;
+ for (int shift = 0; shift < 64; shift += 7) {
+ if (pos >= data.length) {
+ throw malformed("truncated varint");
+ }
+ final byte b = data[pos++];
+ value |= (long) (b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ return value;
+ }
+ }
+ throw malformed("varint exceeds 64 bits");
+ }
+
+ /**
+ * Skips the value of an unrecognized field according to its wire type.
+ *
+ * @param tag The field tag.
+ * @throws InvalidFormatException Thrown if the wire type is unsupported or the value runs past
+ * the input.
+ */
+ private void skip(long tag) throws InvalidFormatException {
+ switch (wireTypeOf(tag)) {
+ case WIRE_VARINT -> varint();
+ case WIRE_FIXED64 -> advance(8);
+ case WIRE_LEN -> pos = lenPayload(tag);
+ case WIRE_FIXED32 -> advance(4);
+ default -> throw malformed("unsupported wire type " + wireTypeOf(tag));
+ }
+ }
+
+ /**
+ * Advances the position by a fixed number of bytes.
+ *
+ * @param count The number of bytes to skip.
+ * @throws InvalidFormatException Thrown if fewer than {@code count} bytes remain.
+ */
+ private void advance(int count) throws InvalidFormatException {
+ if (pos + count > data.length) {
+ throw malformed("truncated field");
+ }
+ pos += count;
+ }
+
+ /**
+ * Creates the exception for malformed input, carrying the current byte position.
+ *
+ * @param detail A short description of what is malformed.
+ * @return The exception to throw.
+ */
+ private InvalidFormatException malformed(String detail) {
+ return new InvalidFormatException(
+ "The model data is malformed at byte " + pos + ": " + detail + ".");
+ }
+
+ /** The fields of a {@code ModelProto} that inference needs, with the proto's defaults. */
+ static final class RawModel {
+
+ static final int TYPE_NORMAL = 1;
+ static final int TYPE_UNKNOWN = 2;
+ static final int TYPE_CONTROL = 3;
+ static final int TYPE_USER_DEFINED = 4;
+ static final int TYPE_UNUSED = 5;
+ static final int TYPE_BYTE = 6;
+
+ static final int MODEL_TYPE_UNIGRAM = 1;
+ static final int MODEL_TYPE_BPE = 2;
+
+ final List pieces = new ArrayList<>();
+ final List scores = new ArrayList<>();
+ final List types = new ArrayList<>();
+
+ int modelType = MODEL_TYPE_UNIGRAM;
+ boolean byteFallback = false;
+ boolean treatWhitespaceAsSuffix = false;
+ int unkId = 0;
+
+ byte[] precompiledCharsMap = new byte[0];
+ boolean addDummyPrefix = true;
+ boolean removeExtraWhitespaces = true;
+ boolean escapeWhitespaces = true;
+
+ final List selfTestInputs = new ArrayList<>();
+ final List selfTestExpected = new ArrayList<>();
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java
new file mode 100644
index 0000000000..42477251ab
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Comparator;
+
+/**
+ * An immutable byte-level trie over vocabulary pieces, packed into flat arrays.
+ *
+ * Encoding walks it one byte at a time ({@link #step(int, byte)}), so every piece that starts
+ * at a given input position is enumerated in one forward pass. Wide nodes dispatch through a
+ * 256-entry direct table and narrow nodes scan a short sorted label slice; both layouts enumerate
+ * identical transitions.
+ */
+final class PieceTrie implements Serializable {
+
+ private static final long serialVersionUID = 30340094783102906L;
+
+ /** The node id returned when no transition exists. */
+ static final int DEAD = -1;
+
+ // A node dispatches through a 256-entry slice of directPool when it has more children than
+ // this; otherwise a linear scan of the sorted label slice is used.
+ private static final int DIRECT_THRESHOLD = 8;
+
+ /** The width of a wide node's direct dispatch slice. */
+ private static final int DIRECT_TABLE_SIZE = 256;
+
+ // Per node: the slice [childStart[n], childStart[n + 1]) of labels/childNodes, and the piece id
+ // accepted at the node, or -1. Wide nodes additionally index directPool at directStart[n].
+ private final int[] childStart;
+ private final byte[] labels;
+ private final int[] childNodes;
+ private final int[] values;
+ private final int[] directStart;
+ private final int[] directPool;
+
+ /**
+ * Wraps the packed arrays produced by {@link Builder} and derives the direct-dispatch tables
+ * for wide nodes.
+ *
+ * @param childStart Per node, the start of its edge slice; one trailing entry marks the end.
+ * @param labels The transition label of every edge.
+ * @param childNodes The target node of every edge, parallel to {@code labels}.
+ * @param values Per node, the accepted piece id, or {@code -1}.
+ */
+ private PieceTrie(int[] childStart, byte[] labels, int[] childNodes, int[] values) {
+ this.childStart = childStart;
+ this.labels = labels;
+ this.childNodes = childNodes;
+ this.values = values;
+ this.directStart = new int[values.length];
+ int wide = 0;
+ for (int node = 0; node < values.length; node++) {
+ if (childStart[node + 1] - childStart[node] > DIRECT_THRESHOLD) {
+ directStart[node] = wide * DIRECT_TABLE_SIZE;
+ wide++;
+ } else {
+ directStart[node] = -1;
+ }
+ }
+ this.directPool = new int[wide * DIRECT_TABLE_SIZE];
+ Arrays.fill(directPool, DEAD);
+ for (int node = 0; node < values.length; node++) {
+ final int direct = directStart[node];
+ if (direct >= 0) {
+ for (int edge = childStart[node]; edge < childStart[node + 1]; edge++) {
+ directPool[direct + (labels[edge] & 0xFF)] = childNodes[edge];
+ }
+ }
+ }
+ }
+
+ /**
+ * Builds a trie from pieces and their ids.
+ *
+ * @param pieces The UTF-8 bytes of each piece; must not be null or contain empty keys.
+ * @param ids The id stored for each piece, parallel to {@code pieces}.
+ * @return The packed trie.
+ * @throws IllegalArgumentException Thrown if a piece is defined more than once.
+ */
+ static PieceTrie build(byte[][] pieces, int[] ids) {
+ final Integer[] order = new Integer[pieces.length];
+ for (int i = 0; i < order.length; i++) {
+ order[i] = i;
+ }
+ Arrays.sort(order, Comparator.comparing(i -> pieces[i], Arrays::compareUnsigned));
+
+ // First pass counts nodes and edges, second pass fills the packed arrays; both walk the
+ // sorted keys with the same recursion, so the shapes agree by construction.
+ final Builder builder = new Builder(pieces, ids, order);
+ builder.count(0, pieces.length, 0);
+ builder.allocate();
+ builder.fill(0, pieces.length, 0);
+ return new PieceTrie(builder.childStart, builder.labels, builder.childNodes, builder.values);
+ }
+
+ /** {@return the root node id} */
+ int root() {
+ return 0;
+ }
+
+ /**
+ * Follows the transition labeled {@code b}.
+ *
+ * @param node The current node id.
+ * @param b The next key byte.
+ * @return The child node id, or {@link #DEAD} when no such transition exists.
+ */
+ int step(int node, byte b) {
+ final int direct = directStart[node];
+ if (direct >= 0) {
+ return directPool[direct + (b & 0xFF)];
+ }
+ final int to = childStart[node + 1];
+ for (int edge = childStart[node]; edge < to; edge++) {
+ if (labels[edge] == b) {
+ return childNodes[edge];
+ }
+ }
+ return DEAD;
+ }
+
+ /**
+ * Returns the piece id accepted at a node.
+ *
+ * @param node The node id.
+ * @return The id, or {@code -1} when the node accepts no piece.
+ */
+ int value(int node) {
+ return values[node];
+ }
+
+ /**
+ * Returns the byte length of the longest piece in this trie that is a prefix of
+ * {@code input[from, inputLength)}.
+ *
+ * @param input The UTF-8 input buffer; must not be null.
+ * @param inputLength The number of valid bytes in {@code input}.
+ * @param from The offset to match from.
+ * @return The matched length in bytes, or zero when no piece matches.
+ */
+ int longestMatch(byte[] input, int inputLength, int from) {
+ int node = root();
+ int longest = 0;
+ for (int i = from; i < inputLength; i++) {
+ node = step(node, input[i]);
+ if (node == DEAD) {
+ break;
+ }
+ if (value(node) >= 0) {
+ longest = i - from + 1;
+ }
+ }
+ return longest;
+ }
+
+ /**
+ * Creates the exception reported wherever a vocabulary piece turns out to be defined twice,
+ * keeping the message identical across all detection sites.
+ *
+ * @param piece The duplicated piece content.
+ * @return The exception to throw.
+ */
+ static IllegalArgumentException duplicatePiece(String piece) {
+ return new IllegalArgumentException("The piece '" + piece + "' is defined more than once.");
+ }
+
+ // Builds the packed form from keys sorted by unsigned byte order. Key ranges sharing a prefix
+ // are contiguous after the sort, so each recursion partitions its range by the byte at the
+ // current depth.
+ private static final class Builder {
+
+ private final byte[][] pieces;
+ private final int[] ids;
+ private final Integer[] order;
+
+ private int nodeCount;
+ private int edgeCount;
+
+ private int[] childStart;
+ private byte[] labels;
+ private int[] childNodes;
+ private int[] values;
+ private int nextNode;
+ private int nextEdge;
+
+ /**
+ * Prepares a builder over the keys and their sort order; {@link #count} and {@link #fill}
+ * perform the actual construction.
+ *
+ * @param pieces The UTF-8 bytes of each piece.
+ * @param ids The id stored for each piece, parallel to {@code pieces}.
+ * @param order The indices of {@code pieces} sorted by unsigned byte order.
+ */
+ Builder(byte[][] pieces, int[] ids, Integer[] order) {
+ this.pieces = pieces;
+ this.ids = ids;
+ this.order = order;
+ }
+
+ /**
+ * Counts the nodes and edges of the subtrie for the sorted key range {@code [from, to)} at the
+ * given depth.
+ *
+ * @param from The inclusive start index into {@code order}.
+ * @param to The exclusive end index into {@code order}.
+ * @param depth The byte depth this node partitions on.
+ * @throws IllegalArgumentException Thrown if a key is defined more than once.
+ */
+ void count(int from, int to, int depth) {
+ nodeCount++;
+ int i = from;
+ if (i < to && pieces[order[i]].length == depth) {
+ i++;
+ // A second key ending at the same depth is a duplicate; the sort made them adjacent.
+ if (i < to && pieces[order[i]].length == depth) {
+ throw duplicatePiece(new String(pieces[order[i]], StandardCharsets.UTF_8));
+ }
+ }
+ while (i < to) {
+ final byte label = pieces[order[i]][depth];
+ int j = i;
+ while (j < to && pieces[order[j]][depth] == label) {
+ j++;
+ }
+ edgeCount++;
+ count(i, j, depth + 1);
+ i = j;
+ }
+ }
+
+ /** Allocates the packed arrays to the node and edge counts gathered by {@link #count}. */
+ void allocate() {
+ childStart = new int[nodeCount + 1];
+ labels = new byte[edgeCount];
+ childNodes = new int[edgeCount];
+ values = new int[nodeCount];
+ }
+
+ /**
+ * Fills the packed arrays for the sorted key range {@code [from, to)} at the given depth and
+ * returns the node id assigned to it.
+ *
+ * @param from The inclusive start index into {@code order}.
+ * @param to The exclusive end index into {@code order}.
+ * @param depth The byte depth this node partitions on.
+ * @return The id of the node created for this range.
+ * @throws IllegalArgumentException Thrown if a key is defined more than once.
+ */
+ int fill(int from, int to, int depth) {
+ final int node = nextNode++;
+ values[node] = -1;
+ int i = from;
+ if (i < to && pieces[order[i]].length == depth) {
+ if (values[node] != -1 || (i + 1 < to && pieces[order[i + 1]].length == depth)) {
+ throw duplicatePiece(new String(pieces[order[i]], StandardCharsets.UTF_8));
+ }
+ values[node] = ids[order[i]];
+ i++;
+ }
+ // Reserve this node's edge slice before recursing so siblings stay contiguous.
+ final int sliceStart = nextEdge;
+ int sliceCount = 0;
+ int scan = i;
+ while (scan < to) {
+ final byte label = pieces[order[scan]][depth];
+ int j = scan;
+ while (j < to && pieces[order[j]][depth] == label) {
+ j++;
+ }
+ sliceCount++;
+ scan = j;
+ }
+ nextEdge += sliceCount;
+ childStart[node] = sliceStart;
+ childStart[node + 1] = nextEdge;
+
+ int edge = sliceStart;
+ while (i < to) {
+ final byte label = pieces[order[i]][depth];
+ int j = i;
+ while (j < to && pieces[order[j]][depth] == label) {
+ j++;
+ }
+ labels[edge] = label;
+ childNodes[edge] = fill(i, j, depth + 1);
+ edge++;
+ i = j;
+ }
+ return node;
+ }
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Segment.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Segment.java
new file mode 100644
index 0000000000..7f603c6ac5
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Segment.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+/**
+ * One encoded piece as a half-open byte range of the normalized text plus its vocabulary id.
+ *
+ * @param from The inclusive start offset in the normalized bytes.
+ * @param to The exclusive end offset in the normalized bytes.
+ * @param id The vocabulary id; the unknown id when no piece covers the range.
+ */
+record Segment(int from, int to, int id) {
+}
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java
new file mode 100644
index 0000000000..5494282e95
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.Serializable;
+
+/**
+ * The model-embedded text normalizer of a SentencePiece model, operating in UTF-8 byte space.
+ *
+ * Normalization applies the model's precompiled character map (leftmost-longest replacement
+ * rules over UTF-8 prefixes), collapses and trims whitespace, optionally prepends the
+ * word-boundary marker, and escapes spaces to U+2581. Alongside the normalized bytes it produces
+ * {@code normToOrig}, mapping every normalized byte to the offset of the original byte chunk it
+ * was derived from, with one trailing entry for the end position; that map is what lets every
+ * downstream piece report an exact span of the caller's text.
+ */
+final class SentencePieceNormalizer implements Serializable {
+
+ private static final long serialVersionUID = -3059745470932191300L;
+
+ // U+2581 LOWER ONE EIGHTH BLOCK in UTF-8, the escaped form of a space.
+ static final byte[] SPACE_SYMBOL = {(byte) 0xE2, (byte) 0x96, (byte) 0x81};
+
+ // U+FFFD REPLACEMENT CHARACTER in UTF-8, emitted for a malformed byte.
+ private static final byte[] REPLACEMENT_CHAR = {(byte) 0xEF, (byte) 0xBF, (byte) 0xBD};
+
+ private final DoubleArrayTrie trie;
+ private final byte[] blob;
+ private final int replacementsFrom;
+ private final boolean addDummyPrefix;
+ private final boolean removeExtraWhitespaces;
+ private final boolean escapeWhitespaces;
+ private final boolean treatWhitespaceAsSuffix;
+ private final PieceTrie userDefinedMatcher;
+ // For each possible first byte, whether any character-map rule or user-defined symbol starts
+ // with it; a clear bit means normalizePrefix passes the byte through raw.
+ private final boolean[] ruleLead = new boolean[256];
+
+ /**
+ * Instantiates the normalizer.
+ *
+ * @param precompiledCharsMap The serialized character map; empty when the model has none.
+ * @param addDummyPrefix Whether a word-boundary marker is prepended.
+ * @param removeExtraWhitespaces Whether leading, trailing, and repeated whitespace collapses.
+ * @param escapeWhitespaces Whether spaces become U+2581.
+ * @param treatWhitespaceAsSuffix Whether the dummy marker is appended instead of prepended.
+ * @param userDefinedMatcher Longest-match trie over user-defined symbols that must pass
+ * through normalization untouched, or null when the model
+ * defines none.
+ * @throws IllegalArgumentException Thrown if the character map is structurally invalid.
+ */
+ SentencePieceNormalizer(byte[] precompiledCharsMap, boolean addDummyPrefix,
+ boolean removeExtraWhitespaces, boolean escapeWhitespaces,
+ boolean treatWhitespaceAsSuffix, PieceTrie userDefinedMatcher) {
+ if (precompiledCharsMap.length == 0) {
+ trie = null;
+ blob = null;
+ replacementsFrom = 0;
+ } else {
+ // Layout: .
+ if (precompiledCharsMap.length <= 4) {
+ throw new IllegalArgumentException("The precompiled character map is truncated.");
+ }
+ final long trieSize = (precompiledCharsMap[0] & 0xFFL)
+ | (precompiledCharsMap[1] & 0xFFL) << 8
+ | (precompiledCharsMap[2] & 0xFFL) << 16
+ | (precompiledCharsMap[3] & 0xFFL) << 24;
+ if (trieSize >= precompiledCharsMap.length - 4) {
+ throw new IllegalArgumentException(
+ "The precompiled character map declares a trie of " + trieSize
+ + " bytes but only " + (precompiledCharsMap.length - 4) + " bytes follow.");
+ }
+ if (trieSize < 1024 || (trieSize & 0x3FF) != 0) {
+ throw new IllegalArgumentException(
+ "The precompiled character map trie size " + trieSize
+ + " is not a positive multiple of 1024.");
+ }
+ if (precompiledCharsMap[precompiledCharsMap.length - 1] != 0) {
+ throw new IllegalArgumentException(
+ "The precompiled character map replacement block is not null-terminated.");
+ }
+ trie = new DoubleArrayTrie(precompiledCharsMap, 4, (int) trieSize);
+ blob = precompiledCharsMap;
+ replacementsFrom = 4 + (int) trieSize;
+ }
+ this.addDummyPrefix = addDummyPrefix;
+ this.removeExtraWhitespaces = removeExtraWhitespaces;
+ this.escapeWhitespaces = escapeWhitespaces;
+ this.treatWhitespaceAsSuffix = treatWhitespaceAsSuffix;
+ this.userDefinedMatcher = userDefinedMatcher;
+ for (int b = 0; b < 256; b++) {
+ final boolean charsMapLead = trie != null && trie.hasTransitionFromRoot(b);
+ final boolean userDefinedLead = userDefinedMatcher != null
+ && userDefinedMatcher.step(userDefinedMatcher.root(), (byte) b) != PieceTrie.DEAD;
+ ruleLead[b] = charsMapLead || userDefinedLead;
+ }
+ }
+
+ /**
+ * The normalized bytes plus the normalized-byte to original-byte offset map. The arrays are
+ * builder-backed and may be oversized; {@code length} bytes are valid, and the offset map
+ * holds {@code length + 1} entries.
+ */
+ record Normalized(byte[] bytes, int length, int[] normToOrig) {
+ }
+
+ /**
+ * One normalization step: {@code consumed} input bytes produced {@code data[from, to)}. The data
+ * array is the input itself (pass-through), the replacement blob, or the replacement character.
+ */
+ private static final class Chunk {
+
+ private byte[] data;
+ private int from;
+ private int to;
+ private int consumed;
+
+ /** {@return whether this chunk is exactly one ASCII space byte} */
+ boolean isSingleSpace() {
+ return to - from == 1 && data[from] == ' ';
+ }
+ }
+
+ /**
+ * Normalizes UTF-8 input.
+ *
+ * @param input The buffer holding well-formed UTF-8 bytes; must not be null.
+ * @param inputLength The number of valid bytes in {@code input}.
+ * @return The normalized bytes with the offset map; the arrays are builder-backed, valid for
+ * {@code length} bytes and {@code length + 1} map entries.
+ */
+ Normalized normalize(byte[] input, int inputLength) {
+ final ByteBuilder normalized = new ByteBuilder(inputLength + (inputLength >> 1) + 4);
+ final IntBuilder normToOrig = new IntBuilder(inputLength + (inputLength >> 1) + 5);
+ final Chunk chunk = new Chunk();
+
+ int from = 0;
+ int consumed = 0;
+
+ // Ignores leading whitespace.
+ if (removeExtraWhitespaces) {
+ while (from < inputLength) {
+ normalizePrefix(input, inputLength, from, chunk);
+ if (!chunk.isSingleSpace()) {
+ break;
+ }
+ from += chunk.consumed;
+ consumed += chunk.consumed;
+ }
+ }
+
+ // All input was whitespace.
+ if (from >= inputLength) {
+ normToOrig.append(consumed);
+ return new Normalized(normalized.array(), 0, normToOrig.array());
+ }
+
+ final byte[] spaceSymbol = escapeWhitespaces ? SPACE_SYMBOL : SINGLE_SPACE;
+
+ if (!treatWhitespaceAsSuffix && addDummyPrefix) {
+ appendSpace(normalized, normToOrig, spaceSymbol, consumed);
+ }
+
+ boolean isPrevSpace = removeExtraWhitespaces;
+ while (from < inputLength) {
+ final int lead = input[from] & 0xFF;
+ // An ASCII byte no rule starts with passes through raw: the chunk is the byte itself, no
+ // leading-space stripping applies, and it does not end in a space.
+ if (lead < 0x80 && lead != ' ' && !ruleLead[lead]) {
+ normalized.append(input[from]);
+ normToOrig.append(consumed);
+ consumed++;
+ from++;
+ isPrevSpace = false;
+ continue;
+ }
+
+ normalizePrefix(input, inputLength, from, chunk);
+ int spFrom = chunk.from;
+ final int spTo = chunk.to;
+ final byte[] spData = chunk.data;
+
+ // Removes leading spaces in the chunk if the previous chunk ended with whitespace.
+ while (isPrevSpace && spFrom < spTo && spData[spFrom] == ' ') {
+ spFrom++;
+ }
+
+ if (spFrom < spTo) {
+ for (int n = spFrom; n < spTo; n++) {
+ if (spData[n] == ' ') {
+ appendSpace(normalized, normToOrig, spaceSymbol, consumed);
+ } else {
+ normalized.append(spData[n]);
+ normToOrig.append(consumed);
+ }
+ }
+ isPrevSpace = spData[spTo - 1] == ' ';
+ }
+
+ consumed += chunk.consumed;
+ from += chunk.consumed;
+ if (!removeExtraWhitespaces) {
+ isPrevSpace = false;
+ }
+ }
+
+ // Ignores trailing whitespace.
+ if (removeExtraWhitespaces) {
+ while (normalized.endsWith(spaceSymbol)) {
+ final int length = normalized.length() - spaceSymbol.length;
+ consumed = normToOrig.get(length);
+ normalized.truncate(length);
+ normToOrig.truncate(length);
+ }
+ }
+
+ if (treatWhitespaceAsSuffix && addDummyPrefix) {
+ appendSpace(normalized, normToOrig, spaceSymbol, consumed);
+ }
+
+ normToOrig.append(consumed);
+ if (normToOrig.length() != normalized.length() + 1) {
+ throw new IllegalStateException("The offset map has " + normToOrig.length()
+ + " entries for " + normalized.length() + " normalized bytes.");
+ }
+ return new Normalized(normalized.array(), normalized.length(), normToOrig.array());
+ }
+
+ private static final byte[] SINGLE_SPACE = {' '};
+
+ /**
+ * Appends the space symbol to the normalized output, mapping each of its bytes to the same
+ * original-byte offset.
+ *
+ * @param normalized The normalized-byte builder to append to.
+ * @param normToOrig The offset-map builder to append to.
+ * @param spaceSymbol The bytes of the (possibly escaped) space symbol.
+ * @param consumed The original-byte offset the symbol maps back to.
+ */
+ private static void appendSpace(ByteBuilder normalized, IntBuilder normToOrig,
+ byte[] spaceSymbol, int consumed) {
+ normalized.append(spaceSymbol, 0, spaceSymbol.length);
+ for (int i = 0; i < spaceSymbol.length; i++) {
+ normToOrig.append(consumed);
+ }
+ }
+
+ /**
+ * Fills the scratch with the normalized form of the longest applicable prefix of
+ * {@code input[from, inputLength)}: a user-defined symbol passes through raw, otherwise the
+ * longest character-map rule applies, otherwise one code point passes through raw (or becomes
+ * U+FFFD when the lead byte is malformed).
+ *
+ * @param input The UTF-8 input buffer.
+ * @param inputLength The number of valid bytes in {@code input}.
+ * @param from The offset to normalize from.
+ * @param chunk The scratch to fill.
+ */
+ private void normalizePrefix(byte[] input, int inputLength, int from, Chunk chunk) {
+ if (userDefinedMatcher != null) {
+ final int matched = userDefinedMatcher.longestMatch(input, inputLength, from);
+ if (matched > 0) {
+ chunk.data = input;
+ chunk.from = from;
+ chunk.to = from + matched;
+ chunk.consumed = matched;
+ return;
+ }
+ }
+
+ if (trie != null) {
+ final long match = trie.longestPrefixMatch(input, from, inputLength);
+ if (match >= 0) {
+ final int value = (int) (match >>> 32);
+ final int length = (int) (match & 0xFFFFFFFFL);
+ final int replacementFrom = replacementsFrom + value;
+ if (replacementFrom < blob.length) {
+ int replacementTo = replacementFrom;
+ while (blob[replacementTo] != 0) {
+ replacementTo++;
+ }
+ chunk.data = blob;
+ chunk.from = replacementFrom;
+ chunk.to = replacementTo;
+ chunk.consumed = length;
+ return;
+ }
+ }
+ }
+
+ final int charLength = Math.min(utf8Length(input[from]), inputLength - from);
+ if (isMalformed(input, from, charLength)) {
+ chunk.data = REPLACEMENT_CHAR;
+ chunk.from = 0;
+ chunk.to = REPLACEMENT_CHAR.length;
+ chunk.consumed = 1;
+ return;
+ }
+ chunk.data = input;
+ chunk.from = from;
+ chunk.to = from + charLength;
+ chunk.consumed = charLength;
+ }
+
+ /**
+ * Returns the byte length of a UTF-8 sequence from its lead byte; trail and malformed lead bytes
+ * report one byte.
+ *
+ * @param lead The lead byte.
+ * @return The sequence length in bytes, from one to four.
+ */
+ static int utf8Length(byte lead) {
+ final int high = (lead & 0xFF) >>> 4;
+ if (high < 0xC) {
+ return 1;
+ }
+ return switch (high) {
+ case 0xC, 0xD -> 2;
+ case 0xE -> 3;
+ default -> 4;
+ };
+ }
+
+ /**
+ * Checks a single code point for well-formedness: correct trail-byte count and no unpaired
+ * surrogate or out-of-range value.
+ *
+ * @param input The UTF-8 input buffer.
+ * @param from The offset of the lead byte.
+ * @param length The candidate sequence length.
+ * @return {@code true} when the sequence is malformed.
+ */
+ private static boolean isMalformed(byte[] input, int from, int length) {
+ if ((input[from] & 0x80) == 0) {
+ return false;
+ }
+ if ((input[from] & 0xC0) == 0x80 || length < utf8Length(input[from])) {
+ return true;
+ }
+ for (int i = from + 1; i < from + length; i++) {
+ if ((input[i] & 0xC0) != 0x80) {
+ return true;
+ }
+ }
+ final int codePoint = codePointAt(input, from, length);
+ return codePoint < 0 || (codePoint >= 0xD800 && codePoint <= 0xDFFF) || codePoint > 0x10FFFF
+ || length != minimalUtf8Length(codePoint);
+ }
+
+ /**
+ * Decodes the code point of a UTF-8 sequence of the given length.
+ *
+ * @param input The UTF-8 input buffer.
+ * @param from The offset of the lead byte.
+ * @param length The sequence length in bytes, from one to four.
+ * @return The decoded code point.
+ */
+ private static int codePointAt(byte[] input, int from, int length) {
+ return switch (length) {
+ case 1 -> input[from] & 0x7F;
+ case 2 -> (input[from] & 0x1F) << 6 | (input[from + 1] & 0x3F);
+ case 3 -> (input[from] & 0x0F) << 12 | (input[from + 1] & 0x3F) << 6
+ | (input[from + 2] & 0x3F);
+ default -> (input[from] & 0x07) << 18 | (input[from + 1] & 0x3F) << 12
+ | (input[from + 2] & 0x3F) << 6 | (input[from + 3] & 0x3F);
+ };
+ }
+
+ /**
+ * Returns the number of bytes the shortest UTF-8 encoding of a code point uses, which detects
+ * overlong encodings.
+ *
+ * @param codePoint The code point.
+ * @return The minimal encoding length in bytes, from one to four.
+ */
+ private static int minimalUtf8Length(int codePoint) {
+ if (codePoint < 0x80) {
+ return 1;
+ }
+ if (codePoint < 0x800) {
+ return 2;
+ }
+ if (codePoint < 0x10000) {
+ return 3;
+ }
+ return 4;
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java
new file mode 100644
index 0000000000..041cd7b285
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java
@@ -0,0 +1,769 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputFilter;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.IntUnaryOperator;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.tokenize.SubwordPiece;
+import opennlp.tools.tokenize.SubwordTokenizer;
+import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.util.normalizer.AlignedText;
+import opennlp.tools.util.normalizer.Alignment;
+import opennlp.tools.util.normalizer.OffsetAwareNormalizer;
+
+/**
+ * A {@link SubwordTokenizer} over a trained SentencePiece {@code .model} file, implemented purely
+ * in Java. The file carries the vocabulary with piece scores and types, the segmentation algorithm
+ * (unigram language model or byte-pair encoding), and the text normalizer, all of which this class
+ * runs.
+ *
+ * Every piece carries the exact span of the caller's original text it came from, mapped back
+ * through the model's own normalizer, which is also exposed through {@link OffsetAwareNormalizer}
+ * for reuse outside tokenization.
+ *
+ * Instances are immutable after loading and safe for concurrent use by multiple threads.
+ *
+ * Beyond {@link #load(Path) loading} the native {@code .model} format, a tokenizer can be
+ * persisted with {@link #serialize(OutputStream)} and read back with
+ * {@link #deserialize(InputStream)}. Reads are guarded by an {@link java.io.ObjectInputFilter}
+ * that allow-lists only the classes reachable from a legitimate tokenizer graph and bounds graph
+ * depth, references, and array length.
+ *
+ * @see SentencePiece
+ * @see Kudo & Richardson (EMNLP 2018),
+ * "SentencePiece: A simple and language independent subword tokenizer and detokenizer for
+ * Neural Text Processing"
+ */
+@ThreadSafe
+public final class SentencePieceTokenizer implements SubwordTokenizer, OffsetAwareNormalizer {
+
+ // Serializable through the OffsetAwareNormalizer contract.
+ private static final long serialVersionUID = -4472058014098085134L;
+
+ /** The segmentation algorithm a model was trained with. */
+ public enum Algorithm {
+ /** Unigram language model, decoded by best-path search. */
+ UNIGRAM,
+ /** Byte-pair encoding, decoded by greedy highest-score merging. */
+ BPE
+ }
+
+ // Piece types of the model format.
+ private static final int TYPE_NORMAL = ModelProtoReader.RawModel.TYPE_NORMAL;
+ private static final int TYPE_UNKNOWN = ModelProtoReader.RawModel.TYPE_UNKNOWN;
+ private static final int TYPE_CONTROL = ModelProtoReader.RawModel.TYPE_CONTROL;
+ private static final int TYPE_USER_DEFINED = ModelProtoReader.RawModel.TYPE_USER_DEFINED;
+ private static final int TYPE_UNUSED = ModelProtoReader.RawModel.TYPE_UNUSED;
+ private static final int TYPE_BYTE = ModelProtoReader.RawModel.TYPE_BYTE;
+
+ private static final int MAX_PIECE_LENGTH = 8000;
+
+ private final Algorithm algorithm;
+ private final String[] pieces;
+ private final float[] scores;
+ private final int[] types;
+ private final int unkId;
+ private final boolean byteFallback;
+ private final Map mainPieces;
+ private final Map reservedPieces;
+ private final int[] byteToId;
+ private final SentencePieceNormalizer normalizer;
+ private final UnigramEncoder unigramEncoder;
+ private final BpeEncoder bpeEncoder;
+ private final List selfTestInputs;
+ private final List selfTestExpected;
+
+ /**
+ * Validates a parsed model and derives the runtime structures: the piece maps, the byte-piece
+ * table, the normalizer, and the encoder matching the model's algorithm.
+ *
+ * @param model The parsed model description.
+ * @throws InvalidFormatException Thrown if the model is structurally invalid.
+ */
+ private SentencePieceTokenizer(ModelProtoReader.RawModel model) throws InvalidFormatException {
+ final int count = model.pieces.size();
+ pieces = model.pieces.toArray(new String[0]);
+ scores = new float[count];
+ types = new int[count];
+ for (int i = 0; i < count; i++) {
+ scores[i] = model.scores.get(i);
+ types[i] = model.types.get(i);
+ }
+ byteFallback = model.byteFallback;
+ algorithm = switch (model.modelType) {
+ case ModelProtoReader.RawModel.MODEL_TYPE_UNIGRAM -> Algorithm.UNIGRAM;
+ case ModelProtoReader.RawModel.MODEL_TYPE_BPE -> Algorithm.BPE;
+ default -> throw new InvalidFormatException(
+ "The model type " + model.modelType + " is not supported; only the unigram and BPE"
+ + " algorithms are.");
+ };
+
+ // Splits the vocabulary the way the reference does: pieces of the normal, user-defined, and
+ // unused types participate in segmentation, all others are reserved ids.
+ mainPieces = new HashMap<>(count * 2);
+ reservedPieces = new HashMap<>();
+ final List userDefined = new ArrayList<>();
+ byteToId = new int[256];
+ Arrays.fill(byteToId, -1);
+ int foundUnkId = -1;
+ float minScore = Float.MAX_VALUE;
+ for (int i = 0; i < count; i++) {
+ final String piece = pieces[i];
+ if (piece.length() >= MAX_PIECE_LENGTH) {
+ throw new InvalidFormatException("The piece with id " + i + " is longer than "
+ + MAX_PIECE_LENGTH + " characters.");
+ }
+ if (piece.indexOf(0) >= 0) {
+ throw new InvalidFormatException(
+ "The piece with id " + i + " contains a null character.");
+ }
+ final boolean isMain =
+ types[i] == TYPE_NORMAL || types[i] == TYPE_USER_DEFINED || types[i] == TYPE_UNUSED;
+ final Map target =
+ isMain || algorithm == Algorithm.BPE ? mainPieces : reservedPieces;
+ if (mainPieces.containsKey(piece) || reservedPieces.containsKey(piece)) {
+ throw new InvalidFormatException(PieceTrie.duplicatePiece(piece).getMessage());
+ }
+ target.put(piece, i);
+ switch (types[i]) {
+ case TYPE_NORMAL -> minScore = Math.min(minScore, scores[i]);
+ case TYPE_USER_DEFINED -> userDefined.add(piece);
+ case TYPE_UNKNOWN -> {
+ if (foundUnkId >= 0) {
+ throw new InvalidFormatException("The model defines more than one unknown piece.");
+ }
+ foundUnkId = i;
+ }
+ case TYPE_BYTE -> {
+ if (!byteFallback) {
+ throw new InvalidFormatException("The model defines the byte piece '" + piece
+ + "' although byte fallback is disabled.");
+ }
+ final int b = parseBytePiece(piece);
+ if (b < 0) {
+ throw new InvalidFormatException("The byte piece '" + piece + "' is invalid.");
+ }
+ byteToId[b] = i;
+ }
+ default -> {
+ // CONTROL and UNUSED need no bookkeeping here.
+ }
+ }
+ }
+ if (foundUnkId < 0) {
+ throw new InvalidFormatException("The model defines no unknown piece.");
+ }
+ unkId = foundUnkId;
+ if (byteFallback) {
+ for (int b = 0; b < 256; b++) {
+ if (byteToId[b] < 0) {
+ throw new InvalidFormatException("The model enables byte fallback but defines no"
+ + " piece for byte " + b + ".");
+ }
+ }
+ }
+
+ final PieceTrie userDefinedMatcher =
+ userDefined.isEmpty() ? null : trieOf(userDefined, id -> 0);
+
+ normalizer = new SentencePieceNormalizer(model.precompiledCharsMap, model.addDummyPrefix,
+ model.removeExtraWhitespaces, model.escapeWhitespaces, model.treatWhitespaceAsSuffix,
+ userDefinedMatcher);
+
+ final boolean[] unusedFlags = new boolean[count];
+ final boolean[] userDefinedFlags = new boolean[count];
+ final boolean[] reservedFlags = new boolean[count];
+ for (int i = 0; i < count; i++) {
+ unusedFlags[i] = types[i] == TYPE_UNUSED;
+ userDefinedFlags[i] = types[i] == TYPE_USER_DEFINED;
+ reservedFlags[i] = types[i] != TYPE_NORMAL && types[i] != TYPE_USER_DEFINED
+ && types[i] != TYPE_UNUSED;
+ }
+
+ if (algorithm == Algorithm.UNIGRAM) {
+ final List mainList = new ArrayList<>(mainPieces.size());
+ final List mainIds = new ArrayList<>(mainPieces.size());
+ for (int i = 0; i < count; i++) {
+ if (!reservedFlags[i]) {
+ mainList.add(pieces[i]);
+ mainIds.add(i);
+ }
+ }
+ final PieceTrie vocabulary = trieOf(mainList, mainIds::get);
+ unigramEncoder = new UnigramEncoder(vocabulary, scores, unusedFlags, userDefinedFlags,
+ minScore, unkId);
+ bpeEncoder = null;
+ } else {
+ unigramEncoder = null;
+ bpeEncoder = new BpeEncoder(mainPieces, scores, unusedFlags, reservedFlags, unkId,
+ userDefinedMatcher);
+ }
+
+ selfTestInputs = List.copyOf(model.selfTestInputs);
+ selfTestExpected = List.copyOf(model.selfTestExpected);
+ }
+
+ /**
+ * Builds a {@link PieceTrie} over the given pieces.
+ *
+ * @param pieceList The pieces to index.
+ * @param idOf Maps a piece's index in {@code pieceList} to the id the trie stores for it.
+ * @return The packed trie.
+ */
+ private static PieceTrie trieOf(List pieceList, IntUnaryOperator idOf) {
+ final byte[][] keys = new byte[pieceList.size()][];
+ final int[] ids = new int[pieceList.size()];
+ for (int i = 0; i < keys.length; i++) {
+ keys[i] = pieceList.get(i).getBytes(StandardCharsets.UTF_8);
+ ids[i] = idOf.applyAsInt(i);
+ }
+ return PieceTrie.build(keys, ids);
+ }
+
+ /**
+ * Loads a model from a file.
+ *
+ * @param modelFile The {@code .model} file to load; must not be null.
+ * @return The ready-to-use tokenizer.
+ * @throws IOException Thrown if the file cannot be read.
+ * @throws InvalidFormatException Thrown if the content is not a valid model.
+ * @throws IllegalArgumentException Thrown if {@code modelFile} is null.
+ */
+ public static SentencePieceTokenizer load(Path modelFile) throws IOException {
+ if (modelFile == null) {
+ throw new IllegalArgumentException("modelFile must not be null");
+ }
+ return new SentencePieceTokenizer(ModelProtoReader.read(Files.readAllBytes(modelFile)));
+ }
+
+ /**
+ * Loads a model from a stream. The stream is read fully but not closed.
+ *
+ * @param in The stream positioned at the start of a {@code .model} serialization; must not be
+ * null.
+ * @return The ready-to-use tokenizer.
+ * @throws IOException Thrown if the stream cannot be read.
+ * @throws InvalidFormatException Thrown if the content is not a valid model.
+ * @throws IllegalArgumentException Thrown if {@code in} is null.
+ */
+ public static SentencePieceTokenizer load(InputStream in) throws IOException {
+ if (in == null) {
+ throw new IllegalArgumentException("in must not be null");
+ }
+ return new SentencePieceTokenizer(ModelProtoReader.read(in.readAllBytes()));
+ }
+
+ /**
+ * Serializes this tokenizer to the given {@link OutputStream} using Java object serialization.
+ * The resulting stream can be read back with {@link #deserialize(InputStream)}.
+ *
+ * @param out The {@link OutputStream} to write to; must not be null.
+ * @throws IOException Thrown if IO errors occurred during serialization.
+ * @throws IllegalArgumentException Thrown if {@code out} is null.
+ */
+ public void serialize(OutputStream out) throws IOException {
+ if (out == null) {
+ throw new IllegalArgumentException("out must not be null");
+ }
+ try (ObjectOutputStream oos = new ObjectOutputStream(out)) {
+ oos.writeObject(this);
+ }
+ }
+
+ /**
+ * Deserializes a {@link SentencePieceTokenizer} from the given {@link InputStream} using
+ * {@link DeserializationLimits#DEFAULT default} resource limits.
+ *
+ * The stream is filtered via an {@link ObjectInputFilter} that allow-lists only the classes
+ * required to reconstruct a {@link SentencePieceTokenizer}, plus resource limits on graph depth,
+ * references, and array length. Foreign payloads are rejected with
+ * {@link java.io.InvalidClassException} before {@link ObjectInputStream#readObject()}
+ * returns.
+ *
+ * Callers should still treat this method as defense-in-depth: only invoke it on streams from
+ * trusted sources. If the default limits are too tight for an unusually large model, use
+ * {@link #deserialize(InputStream, DeserializationLimits)} to supply higher limits. The class
+ * allow-list is intentionally not configurable; loosening it would defeat the purpose of the
+ * filter.
+ *
+ * @param in The {@link InputStream} to read from; must not be null.
+ * @return The reconstructed tokenizer.
+ * @throws IOException Thrown if IO errors occurred during deserialization, including
+ * {@link java.io.InvalidClassException} when the stream contains a class outside the
+ * allow-list or exceeds a resource limit.
+ * @throws ClassNotFoundException Thrown if required classes are not found.
+ * @throws IllegalArgumentException Thrown if {@code in} is null.
+ */
+ public static SentencePieceTokenizer deserialize(InputStream in)
+ throws IOException, ClassNotFoundException {
+ return deserialize(in, DeserializationLimits.DEFAULT);
+ }
+
+ /**
+ * Deserializes a {@link SentencePieceTokenizer} from the given {@link InputStream} using the
+ * supplied {@link DeserializationLimits resource limits}.
+ *
+ * Use this overload when the {@link DeserializationLimits#DEFAULT default limits} reject a
+ * legitimate model, for example one with a very large vocabulary. The class allow-list applied
+ * to the stream is the same as for {@link #deserialize(InputStream)}; only the numeric limits
+ * change.
+ *
+ * @param in The {@link InputStream} to read from; must not be null.
+ * @param limits The {@link DeserializationLimits} to apply; must not be null.
+ * @return The reconstructed tokenizer.
+ * @throws IOException Thrown if IO errors occurred during deserialization, including
+ * {@link java.io.InvalidClassException} when the stream contains a class outside the
+ * allow-list or exceeds one of the supplied limits.
+ * @throws ClassNotFoundException Thrown if required classes are not found.
+ * @throws IllegalArgumentException Thrown if {@code in} or {@code limits} is null.
+ */
+ public static SentencePieceTokenizer deserialize(InputStream in, DeserializationLimits limits)
+ throws IOException, ClassNotFoundException {
+ if (in == null) {
+ throw new IllegalArgumentException("in must not be null");
+ }
+ if (limits == null) {
+ throw new IllegalArgumentException("limits must not be null");
+ }
+ try (ObjectInputStream ois = new ObjectInputStream(in)) {
+ ois.setObjectInputFilter(buildFilter(limits));
+ return (SentencePieceTokenizer) ois.readObject();
+ }
+ }
+
+ /**
+ * Resource limits applied by the {@link ObjectInputFilter} used by
+ * {@link SentencePieceTokenizer#deserialize(InputStream, DeserializationLimits)}.
+ *
+ * The limits bound graph traversal regardless of the class allow-list and provide
+ * defense-in-depth against pathological streams. The {@linkplain #DEFAULT default values} are
+ * generous enough for typical production models; raise them only if a legitimate model is
+ * rejected.
+ *
+ * @param maxDepth Maximum object-graph nesting depth. Must be {@code > 0}.
+ * @param maxRefs Maximum number of internal references the stream may create.
+ * Must be {@code > 0}.
+ * @param maxArrayLength Maximum length of any single array allocation requested by the stream.
+ * Must be {@code > 0}.
+ */
+ public record DeserializationLimits(long maxDepth, long maxRefs, long maxArrayLength) {
+
+ /**
+ * Default limits. Sized so that models with vocabularies of several hundred thousand pieces
+ * round-trip while pathological streams stay bounded.
+ */
+ public static final DeserializationLimits DEFAULT =
+ new DeserializationLimits(MAX_DEPTH_DEFAULT, MAX_REFS_DEFAULT, MAX_ARRAY_DEFAULT);
+
+ /**
+ * Validates the limits.
+ *
+ * @throws IllegalArgumentException Thrown if any of {@code maxDepth}, {@code maxRefs}, or
+ * {@code maxArrayLength} is {@code <= 0}.
+ */
+ public DeserializationLimits {
+ if (maxDepth <= 0) {
+ throw new IllegalArgumentException("maxDepth must be > 0");
+ }
+ if (maxRefs <= 0) {
+ throw new IllegalArgumentException("maxRefs must be > 0");
+ }
+ if (maxArrayLength <= 0) {
+ throw new IllegalArgumentException("maxArrayLength must be > 0");
+ }
+ }
+ }
+
+ private static final long MAX_DEPTH_DEFAULT = 64;
+ private static final long MAX_REFS_DEFAULT = 5_000_000;
+ private static final long MAX_ARRAY_DEFAULT = 10_000_000;
+
+ // Allow-list of fully qualified class names that may appear in the serialized graph of a
+ // SentencePieceTokenizer. Anything else is rejected.
+ private static final Set ALLOWED_CLASSES = Set.of(
+ "opennlp.subword.sentencepiece.SentencePieceTokenizer",
+ "opennlp.subword.sentencepiece.SentencePieceTokenizer$Algorithm",
+ "opennlp.subword.sentencepiece.SentencePieceNormalizer",
+ "opennlp.subword.sentencepiece.UnigramEncoder",
+ "opennlp.subword.sentencepiece.BpeEncoder",
+ "opennlp.subword.sentencepiece.PieceTrie",
+ "opennlp.subword.sentencepiece.DoubleArrayTrie",
+ // JDK types used in field declarations. ObjectInputStream invokes the filter for every
+ // class descriptor in the inheritance chain, not only for the runtime class - so the
+ // abstract superclasses java.lang.Number (super of Integer) and java.lang.Enum (super of
+ // Algorithm) must be allow-listed even though no instance of either appears in the stream.
+ "java.lang.String",
+ "java.lang.Number",
+ "java.lang.Integer",
+ "java.lang.Enum",
+ "java.util.HashMap",
+ // HashMap.readObject() requests permission to allocate a Map.Entry[] before reading
+ // entries; the array type itself never appears as a value in the stream.
+ "java.util.Map$Entry",
+ // The unmodifiable lists created by List.copyOf serialize through the CollSer proxy,
+ // which requests an Object[] allocation for the elements, and the filter is also invoked
+ // for the concrete list class the proxy resolves to.
+ "java.util.CollSer",
+ "java.util.ImmutableCollections$List12",
+ "java.util.ImmutableCollections$ListN",
+ "java.lang.Object"
+ );
+
+ /**
+ * Builds the {@link ObjectInputFilter} enforcing the class allow-list and the given limits.
+ *
+ * @param limits The resource limits to enforce; never null here.
+ * @return The filter to install on the reading {@link ObjectInputStream}.
+ */
+ private static ObjectInputFilter buildFilter(DeserializationLimits limits) {
+ return info -> {
+ if (info.depth() > limits.maxDepth()
+ || info.references() > limits.maxRefs()
+ || info.arrayLength() > limits.maxArrayLength()) {
+ return ObjectInputFilter.Status.REJECTED;
+ }
+
+ final Class serialClass = info.serialClass();
+ if (serialClass == null) {
+ return ObjectInputFilter.Status.UNDECIDED;
+ }
+
+ Class componentType = serialClass;
+ while (componentType.isArray()) {
+ componentType = componentType.getComponentType();
+ }
+ if (componentType.isPrimitive()) {
+ return ObjectInputFilter.Status.ALLOWED;
+ }
+ return ALLOWED_CLASSES.contains(componentType.getName())
+ ? ObjectInputFilter.Status.ALLOWED
+ : ObjectInputFilter.Status.REJECTED;
+ };
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public List encode(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("text must not be null");
+ }
+ final Utf8Text input = Utf8Text.of(text);
+ final SentencePieceNormalizer.Normalized normalized =
+ normalizer.normalize(input.bytes(), input.byteLength());
+ final List segments = algorithm == Algorithm.UNIGRAM
+ ? unigramEncoder.encode(normalized.bytes(), normalized.length())
+ : bpeEncoder.encode(normalized.bytes(), normalized.length());
+
+ final List out = new ArrayList<>(segments.size());
+ final byte[] norm = normalized.bytes();
+ final int[] normToOrig = normalized.normToOrig();
+
+ // Accumulates a run of adjacent unknown pieces into one, as the reference does, so a decoder
+ // sees a single unknown token per unknown region.
+ StringBuilder pendingUnk = null;
+ int pendingUnkStart = 0;
+ int pendingUnkEnd = 0;
+
+ for (final Segment segment : segments) {
+ final boolean isUnk = segment.id() == unkId;
+ final boolean isControl = types[segment.id()] == TYPE_CONTROL;
+ // A non-unknown segment reuses its vocabulary string; only unknown segments need decoding.
+ final String piece = isUnk
+ ? new String(norm, segment.from(), segment.to() - segment.from(), StandardCharsets.UTF_8)
+ : pieces[segment.id()];
+
+ if (isControl) {
+ if (pendingUnk != null) {
+ out.add(new SubwordPiece(pendingUnk.toString(), unkId, pendingUnkStart, pendingUnkEnd));
+ pendingUnk = null;
+ }
+ final int at = input.charOffset(normToOrig[segment.from()]);
+ out.add(new SubwordPiece(piece, segment.id(), at, at));
+ continue;
+ }
+
+ final int origBegin = input.charOffset(normToOrig[segment.from()]);
+ final int origEnd = input.charOffset(normToOrig[segment.to()]);
+
+ if (isUnk && byteFallback) {
+ if (pendingUnk != null) {
+ out.add(new SubwordPiece(pendingUnk.toString(), unkId, pendingUnkStart, pendingUnkEnd));
+ pendingUnk = null;
+ }
+ // Decomposes the unknown region into byte pieces; the last one carries the surface span.
+ for (int i = segment.from(); i < segment.to(); i++) {
+ final int b = norm[i] & 0xFF;
+ final boolean last = i == segment.to() - 1;
+ out.add(new SubwordPiece(BYTE_PIECES[b], byteToId[b], origBegin,
+ last ? origEnd : origBegin));
+ }
+ } else if (isUnk) {
+ if (pendingUnk == null) {
+ pendingUnk = new StringBuilder(piece);
+ pendingUnkStart = origBegin;
+ } else {
+ pendingUnk.append(piece);
+ }
+ pendingUnkEnd = origEnd;
+ } else {
+ if (pendingUnk != null) {
+ out.add(new SubwordPiece(pendingUnk.toString(), unkId, pendingUnkStart, pendingUnkEnd));
+ pendingUnk = null;
+ }
+ out.add(new SubwordPiece(piece, segment.id(), origBegin, origEnd));
+ }
+ }
+ if (pendingUnk != null) {
+ out.add(new SubwordPiece(pendingUnk.toString(), unkId, pendingUnkStart, pendingUnkEnd));
+ }
+ return out;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public CharSequence normalize(CharSequence text) {
+ return normalizeAligned(text).normalized();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public AlignedText normalizeAligned(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("text must not be null");
+ }
+ final Utf8Text input = Utf8Text.of(text);
+ final SentencePieceNormalizer.Normalized result =
+ normalizer.normalize(input.bytes(), input.byteLength());
+ final String normalized = new String(result.bytes(), 0, result.length(),
+ StandardCharsets.UTF_8);
+ final int[] normToOrig = result.normToOrig();
+ final byte[] norm = result.bytes();
+
+ // Walks the normalized code points, grouping neighbors that came from the same original
+ // block into one replace run; gaps between blocks are deletions.
+ final Alignment.Builder builder = new Alignment.Builder();
+ int cursor = 0;
+ int groupOrigStart = -1;
+ int groupOrigEnd = -1;
+ int groupChars = 0;
+ int b = 0;
+ final int normLength = result.length();
+ while (b < normLength) {
+ final int byteLength = Math.min(SentencePieceNormalizer.utf8Length(norm[b]),
+ normLength - b);
+ final int origStart = input.charOffset(normToOrig[b]);
+ final int origEnd = input.charOffset(normToOrig[b + byteLength]);
+ final int chars = byteLength == 4 ? 2 : 1;
+ if (groupChars > 0 && origStart == groupOrigStart && origEnd == groupOrigEnd) {
+ groupChars += chars;
+ } else {
+ cursor = flushGroup(builder, cursor, groupOrigStart, groupOrigEnd, groupChars);
+ groupOrigStart = origStart;
+ groupOrigEnd = origEnd;
+ groupChars = chars;
+ }
+ b += byteLength;
+ }
+ cursor = flushGroup(builder, cursor, groupOrigStart, groupOrigEnd, groupChars);
+ if (cursor < input.charLength()) {
+ builder.replace(input.charLength() - cursor, 0);
+ }
+ return new AlignedText(text, normalized, builder.build(input.charLength()));
+ }
+
+ /**
+ * Emits the pending alignment group as a replace run, preceded by a deletion for any original
+ * text skipped before it, and returns the advanced cursor.
+ *
+ * @param builder The alignment builder to append to.
+ * @param cursor The original-text offset reached so far.
+ * @param origStart The inclusive original-text start of the group.
+ * @param origEnd The exclusive original-text end of the group.
+ * @param chars The number of normalized chars in the group; zero flushes nothing.
+ * @return The original-text offset after the group.
+ */
+ private static int flushGroup(Alignment.Builder builder, int cursor, int origStart, int origEnd,
+ int chars) {
+ if (chars == 0) {
+ return cursor;
+ }
+ if (origStart > cursor) {
+ builder.replace(origStart - cursor, 0);
+ }
+ builder.replace(origEnd - Math.max(origStart, cursor), chars);
+ return Math.max(origEnd, cursor);
+ }
+
+ /** {@return the segmentation algorithm of the loaded model} */
+ public Algorithm algorithm() {
+ return algorithm;
+ }
+
+ /** {@return the number of pieces in the vocabulary} */
+ public int vocabularySize() {
+ return pieces.length;
+ }
+
+ /**
+ * Returns the piece string of an id.
+ *
+ * @param id A vocabulary id in {@code [0, vocabularySize())}.
+ * @return The piece string.
+ * @throws IllegalArgumentException Thrown if {@code id} is out of range.
+ */
+ public String idToPiece(int id) {
+ checkId(id);
+ return pieces[id];
+ }
+
+ /**
+ * Returns the id of a piece string.
+ *
+ * @param piece The piece to look up; must not be null.
+ * @return The id, or the unknown id when the vocabulary does not contain the piece.
+ */
+ public int pieceToId(String piece) {
+ if (piece == null) {
+ throw new IllegalArgumentException("piece must not be null");
+ }
+ final Integer reserved = reservedPieces.get(piece);
+ if (reserved != null) {
+ return reserved;
+ }
+ return mainPieces.getOrDefault(piece, unkId);
+ }
+
+ /**
+ * Returns the score of a piece.
+ *
+ * @param id A vocabulary id in {@code [0, vocabularySize())}.
+ * @return The score; a log-probability for unigram models, a merge rank for BPE models.
+ * @throws IllegalArgumentException Thrown if {@code id} is out of range.
+ */
+ public float score(int id) {
+ checkId(id);
+ return scores[id];
+ }
+
+ /** {@return the id of the unknown piece} */
+ public int unknownId() {
+ return unkId;
+ }
+
+ /**
+ * Checks whether an id is the unknown piece.
+ *
+ * @param id A vocabulary id in {@code [0, vocabularySize())}.
+ * @return {@code true} for the unknown piece.
+ * @throws IllegalArgumentException Thrown if {@code id} is out of range.
+ */
+ public boolean isUnknown(int id) {
+ checkId(id);
+ return types[id] == TYPE_UNKNOWN;
+ }
+
+ /**
+ * Checks whether an id is a control piece.
+ *
+ * @param id A vocabulary id in {@code [0, vocabularySize())}.
+ * @return {@code true} for control pieces.
+ * @throws IllegalArgumentException Thrown if {@code id} is out of range.
+ */
+ public boolean isControl(int id) {
+ checkId(id);
+ return types[id] == TYPE_CONTROL;
+ }
+
+ /**
+ * Checks whether an id is a byte-fallback piece.
+ *
+ * @param id A vocabulary id in {@code [0, vocabularySize())}.
+ * @return {@code true} for byte pieces.
+ * @throws IllegalArgumentException Thrown if {@code id} is out of range.
+ */
+ public boolean isByte(int id) {
+ checkId(id);
+ return types[id] == TYPE_BYTE;
+ }
+
+ /**
+ * Verifies that an id is a valid vocabulary id.
+ *
+ * @param id The id to check.
+ * @throws IllegalArgumentException Thrown if {@code id} is outside {@code [0, vocabularySize())}.
+ */
+ private void checkId(int id) {
+ if (id < 0 || id >= pieces.length) {
+ throw new IllegalArgumentException(
+ "The id " + id + " is outside [0, " + pieces.length + ").");
+ }
+ }
+
+ /** {@return the embedded self-test input samples} */
+ List selfTestInputs() {
+ return selfTestInputs;
+ }
+
+ /** {@return the embedded self-test expected segmentations} */
+ List selfTestExpected() {
+ return selfTestExpected;
+ }
+
+ // The prefix of a byte-fallback piece string; a full piece has the form "<0xAB>".
+ private static final String BYTE_PIECE_PREFIX = "<0x";
+
+ // "<0xAB>" piece strings for all byte values, as byte fallback emits them.
+ private static final String[] BYTE_PIECES = new String[256];
+
+ static {
+ final char[] hex = "0123456789ABCDEF".toCharArray();
+ for (int b = 0; b < 256; b++) {
+ BYTE_PIECES[b] = BYTE_PIECE_PREFIX + hex[b >>> 4] + hex[b & 0xF] + ">";
+ }
+ }
+
+ /**
+ * Parses a byte-fallback piece string of the form {@code <0xAB>} into its byte value.
+ *
+ * @param piece The piece string.
+ * @return The byte value in {@code [0, 255]}, or {@code -1} when the string is not a byte piece.
+ */
+ private static int parseBytePiece(String piece) {
+ if (piece.length() != 6 || !piece.startsWith(BYTE_PIECE_PREFIX) || piece.charAt(5) != '>') {
+ return -1;
+ }
+ final int high = Character.digit(piece.charAt(3), 16);
+ final int low = Character.digit(piece.charAt(4), 16);
+ return high < 0 || low < 0 ? -1 : (high << 4) | low;
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java
new file mode 100644
index 0000000000..5be10c2c84
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Viterbi segmentation under a unigram language model: of all ways to cover the normalized text
+ * with vocabulary pieces, it finds the one with the highest total log-probability.
+ *
+ * Characters no piece covers fall back to the unknown id with a fixed penalty below the lowest
+ * piece score, and user-defined symbols receive a length-based bonus score so they always win.
+ */
+final class UnigramEncoder implements Serializable {
+
+ private static final long serialVersionUID = 5648005733414803707L;
+
+ private static final float UNK_PENALTY = 10.0f;
+ // The score of a user-defined symbol is this bonus per matched byte beyond the first instead
+ // of a trained log-probability, so longer user-defined matches always win the best path.
+ private static final float USER_DEFINED_LENGTH_BONUS = 0.1f;
+ private static final float SCORE_RESET_THRESHOLD = 100000.0f;
+
+ private final PieceTrie trie;
+ private final float[] scores;
+ private final boolean[] unused;
+ private final boolean[] userDefined;
+ private final float unkScore;
+ private final int unkId;
+
+ /**
+ * Instantiates the encoder.
+ *
+ * @param trie The vocabulary trie over all matchable pieces.
+ * @param scores The log-probability score of every piece, indexed by id.
+ * @param unused Whether each id has the unused piece type.
+ * @param userDefined Whether each id is a user-defined symbol.
+ * @param minScore The lowest score among normal pieces.
+ * @param unkId The id of the unknown piece.
+ */
+ UnigramEncoder(PieceTrie trie, float[] scores, boolean[] unused, boolean[] userDefined,
+ float minScore, int unkId) {
+ this.trie = trie;
+ this.scores = scores;
+ this.unused = unused;
+ this.userDefined = userDefined;
+ this.unkScore = minScore - UNK_PENALTY;
+ this.unkId = unkId;
+ }
+
+ /**
+ * Segments normalized text.
+ *
+ * @param normalized The buffer holding the normalized UTF-8 bytes; must not be null.
+ * @param size The number of valid bytes in {@code normalized}.
+ * @return The best-path segments covering all bytes, in text order.
+ * @throws IllegalArgumentException Thrown if {@code normalized} is null.
+ */
+ List encode(byte[] normalized, int size) {
+ if (normalized == null) {
+ throw new IllegalArgumentException("normalized must not be null");
+ }
+ if (size == 0) {
+ return List.of();
+ }
+
+ // The best path ending at each byte position (exclusive end), interleaved as
+ // [startsAt, scoreBits, id] triples; scores travel as raw float bits, a lossless round trip.
+ final int[] best = new int[3 * (size + 1)];
+ for (int i = 0; i <= size; i++) {
+ best[3 * i] = -1;
+ }
+ best[1] = Float.floatToRawIntBits(0.0f);
+
+ int startsAt = 0;
+ int maxFrontier = 0;
+ while (startsAt < size) {
+ float bestScoreTillHere = Float.intBitsToFloat(best[3 * startsAt + 1]);
+ if (bestScoreTillHere < -SCORE_RESET_THRESHOLD
+ || bestScoreTillHere > SCORE_RESET_THRESHOLD) {
+ // Re-bases accumulated scores to keep float precision on very long inputs; every
+ // reachable frontier position shifts by the same offset, so the argmax is unchanged.
+ final float offset = bestScoreTillHere;
+ for (int i = startsAt; i <= maxFrontier; i++) {
+ if (i == startsAt || best[3 * i] != -1) {
+ best[3 * i + 1] = Float.floatToRawIntBits(
+ Float.intBitsToFloat(best[3 * i + 1]) - offset);
+ }
+ }
+ bestScoreTillHere = 0.0f;
+ }
+
+ boolean hasSingleNode = false;
+ final int mblen = Math.min(SentencePieceNormalizer.utf8Length(normalized[startsAt]),
+ size - startsAt);
+
+ int node = trie.root();
+ for (int keyPos = startsAt; keyPos < size; ) {
+ node = trie.step(node, normalized[keyPos]);
+ if (node == PieceTrie.DEAD) {
+ break;
+ }
+ keyPos++;
+ final int id = trie.value(node);
+ if (id < 0) {
+ continue;
+ }
+ if (unused[id]) {
+ continue;
+ }
+ maxFrontier = Math.max(maxFrontier, keyPos);
+ final int length = keyPos - startsAt;
+ // User-defined symbols receive a length bonus instead of a trained score.
+ final float score = userDefined[id]
+ ? USER_DEFINED_LENGTH_BONUS * (length - 1) : scores[id];
+ final float candidate = score + bestScoreTillHere;
+ final int slot = 3 * keyPos;
+ if (best[slot] == -1 || candidate > Float.intBitsToFloat(best[slot + 1])) {
+ best[slot + 1] = Float.floatToRawIntBits(candidate);
+ best[slot] = startsAt;
+ best[slot + 2] = id;
+ }
+ if (!hasSingleNode && length == mblen) {
+ hasSingleNode = true;
+ }
+ }
+
+ if (!hasSingleNode) {
+ final int end = startsAt + mblen;
+ maxFrontier = Math.max(maxFrontier, end);
+ final float candidate = unkScore + bestScoreTillHere;
+ final int slot = 3 * end;
+ if (best[slot] == -1 || candidate > Float.intBitsToFloat(best[slot + 1])) {
+ best[slot + 1] = Float.floatToRawIntBits(candidate);
+ best[slot] = startsAt;
+ best[slot + 2] = unkId;
+ }
+ }
+
+ startsAt += mblen;
+ }
+
+ final List results = new ArrayList<>(size / 4 + 1);
+ int endsAt = size;
+ while (endsAt > 0) {
+ final int from = best[3 * endsAt];
+ if (from < 0) {
+ throw new IllegalStateException(
+ "The Viterbi path is broken at normalized byte " + endsAt + ".");
+ }
+ results.add(new Segment(from, endsAt, best[3 * endsAt + 2]));
+ endsAt = from;
+ }
+ Collections.reverse(results);
+ return results;
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java
new file mode 100644
index 0000000000..4f2bd95f0f
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+/**
+ * A caller's text encoded as UTF-8, keeping the map from every byte offset back to the UTF-16
+ * offset it came from.
+ *
+ * The pipeline runs in UTF-8 byte space, but the spans reported to the caller must be UTF-16
+ * offsets into the original {@code CharSequence}; this map converts them. An unpaired surrogate,
+ * which UTF-8 cannot represent, is encoded as U+FFFD.
+ */
+final class Utf8Text {
+
+ private final byte[] bytes;
+ private final int byteLength;
+ // Null for pure-ASCII text, where byte offsets equal UTF-16 offsets.
+ private final int[] byteToChar;
+ private final int charLength;
+
+ /**
+ * Wraps an encoded buffer and its offset map.
+ *
+ * @param bytes The UTF-8 buffer.
+ * @param byteLength The number of valid bytes in {@code bytes}.
+ * @param byteToChar The byte-to-UTF-16 offset map, or null for pure-ASCII text.
+ * @param charLength The length of the original text in UTF-16 units.
+ */
+ private Utf8Text(byte[] bytes, int byteLength, int[] byteToChar, int charLength) {
+ this.bytes = bytes;
+ this.byteLength = byteLength;
+ this.byteToChar = byteToChar;
+ this.charLength = charLength;
+ }
+
+ /**
+ * Encodes text.
+ *
+ * @param text The text to encode; must not be null.
+ * @return The encoded view.
+ */
+ static Utf8Text of(CharSequence text) {
+ final int charLength = text.length();
+ // The common case: pure ASCII, where the bytes are the chars and the map is the identity.
+ int ascii = 0;
+ while (ascii < charLength && text.charAt(ascii) < 0x80) {
+ ascii++;
+ }
+ if (ascii == charLength) {
+ final byte[] exact = new byte[charLength];
+ for (int i = 0; i < charLength; i++) {
+ exact[i] = (byte) text.charAt(i);
+ }
+ return new Utf8Text(exact, charLength, null, charLength);
+ }
+
+ final byte[] bytes = new byte[charLength * 3 + 1];
+ final int[] byteToChar = new int[charLength * 3 + 2];
+ int b = 0;
+ int c = 0;
+ while (c < charLength) {
+ int codePoint = text.charAt(c);
+ int charCount = 1;
+ if (Character.isHighSurrogate((char) codePoint) && c + 1 < charLength
+ && Character.isLowSurrogate(text.charAt(c + 1))) {
+ codePoint = Character.toCodePoint((char) codePoint, text.charAt(c + 1));
+ charCount = 2;
+ } else if (Character.isSurrogate((char) codePoint)) {
+ // An unpaired surrogate has no UTF-8 form; U+FFFD keeps the encoding total.
+ codePoint = 0xFFFD;
+ }
+ final int start = b;
+ if (codePoint < 0x80) {
+ bytes[b++] = (byte) codePoint;
+ } else if (codePoint < 0x800) {
+ bytes[b++] = (byte) (0xC0 | codePoint >>> 6);
+ bytes[b++] = (byte) (0x80 | codePoint & 0x3F);
+ } else if (codePoint < 0x10000) {
+ bytes[b++] = (byte) (0xE0 | codePoint >>> 12);
+ bytes[b++] = (byte) (0x80 | codePoint >>> 6 & 0x3F);
+ bytes[b++] = (byte) (0x80 | codePoint & 0x3F);
+ } else {
+ bytes[b++] = (byte) (0xF0 | codePoint >>> 18);
+ bytes[b++] = (byte) (0x80 | codePoint >>> 12 & 0x3F);
+ bytes[b++] = (byte) (0x80 | codePoint >>> 6 & 0x3F);
+ bytes[b++] = (byte) (0x80 | codePoint & 0x3F);
+ }
+ for (int i = start; i < b; i++) {
+ byteToChar[i] = c;
+ }
+ c += charCount;
+ }
+ byteToChar[b] = charLength;
+ return new Utf8Text(bytes, b, byteToChar, charLength);
+ }
+
+ /** {@return the UTF-8 buffer; valid up to {@link #byteLength()}} */
+ byte[] bytes() {
+ return bytes;
+ }
+
+ /** {@return the number of valid bytes in {@link #bytes()}} */
+ int byteLength() {
+ return byteLength;
+ }
+
+ /** {@return the length of the original text in UTF-16 units} */
+ int charLength() {
+ return charLength;
+ }
+
+ /**
+ * Maps a byte offset to the UTF-16 offset of the character containing it.
+ *
+ * @param byteOffset An offset in {@code [0, bytes().length]}.
+ * @return The UTF-16 offset; the text length for the end offset.
+ */
+ int charOffset(int byteOffset) {
+ return byteToChar == null ? byteOffset : byteToChar[byteOffset];
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java
new file mode 100644
index 0000000000..adaf2a292e
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * The trie's transition function held against a naive map-backed reference over randomized
+ * vocabularies, so the hybrid direct-table and linear-scan node layouts are proven to enumerate
+ * identical transitions; the encoder's correctness rests on that equivalence.
+ */
+class PieceTrieTest {
+
+ @Test
+ void testStepsMatchAMapBackedReferenceOverRandomVocabularies() {
+ final Random random = new Random(7);
+ for (int round = 0; round < 20; round++) {
+ // Piece count crosses the direct-table threshold in both directions, so wide and narrow
+ // roots are both exercised.
+ final int pieceCount = 2 + random.nextInt(60);
+ final Set keys = new HashSet<>();
+ while (keys.size() < pieceCount) {
+ final int length = 1 + random.nextInt(5);
+ final StringBuilder key = new StringBuilder();
+ for (int i = 0; i < length; i++) {
+ key.append((char) ('a' + random.nextInt(random.nextBoolean() ? 26 : 4)));
+ }
+ keys.add(key.toString());
+ }
+ final byte[][] pieces = new byte[keys.size()][];
+ final int[] ids = new int[keys.size()];
+ final Map reference = new HashMap<>();
+ int index = 0;
+ for (final String key : keys) {
+ pieces[index] = key.getBytes(StandardCharsets.UTF_8);
+ ids[index] = index;
+ reference.put(key, index);
+ index++;
+ }
+ final PieceTrie trie = PieceTrie.build(pieces, ids);
+
+ // Every walk over random query strings must accept exactly the reference's prefixes.
+ for (int query = 0; query < 200; query++) {
+ final int length = 1 + random.nextInt(8);
+ final StringBuilder text = new StringBuilder();
+ for (int i = 0; i < length; i++) {
+ text.append((char) ('a' + random.nextInt(6)));
+ }
+ int node = trie.root();
+ for (int i = 0; i < length; i++) {
+ node = trie.step(node, (byte) text.charAt(i));
+ final String prefix = text.substring(0, i + 1);
+ final boolean anyKeyHasPrefix =
+ keys.stream().anyMatch(k -> k.startsWith(prefix));
+ if (node == PieceTrie.DEAD) {
+ assertFalse(anyKeyHasPrefix, "dead end despite live prefix: " + prefix);
+ break;
+ }
+ final Integer expected = reference.get(prefix);
+ assertEquals(expected == null ? -1 : expected, trie.value(node),
+ "value mismatch at prefix: " + prefix);
+ }
+ }
+ }
+ }
+
+ @Test
+ void testWideRootDispatchesAllByteValues() {
+ // 200 distinct single-byte pieces force the direct-table layout at the root.
+ final byte[][] pieces = new byte[200][];
+ final int[] ids = new int[200];
+ for (int i = 0; i < 200; i++) {
+ pieces[i] = new byte[] {(byte) (i + 20)};
+ ids[i] = i;
+ }
+ final PieceTrie trie = PieceTrie.build(pieces, ids);
+ for (int b = 0; b < 256; b++) {
+ final int node = trie.step(trie.root(), (byte) b);
+ if (b >= 20 && b < 220) {
+ assertEquals(b - 20, trie.value(node));
+ } else {
+ assertEquals(PieceTrie.DEAD, node);
+ }
+ }
+ }
+
+ @Test
+ void testDuplicatePiecesFailLoudly() {
+ final byte[][] pieces = {{'a'}, {'a'}};
+ assertThrows(IllegalArgumentException.class, () -> PieceTrie.build(pieces, new int[] {0, 1}));
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java
new file mode 100644
index 0000000000..7ac28e67ba
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.tokenize.SubwordPiece;
+import opennlp.tools.util.Span;
+import opennlp.tools.util.normalizer.AlignedText;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Exercises the {@code OffsetAwareNormalizer} view: the model normalizer's output must align
+ * back to the original text exactly, including through whitespace collapsing, character-map
+ * replacements, and supplementary characters.
+ */
+class SentencePieceAlignmentTest {
+
+ private static SentencePieceTokenizer unigram() {
+ return SentencePieceFixtures.tokenizer("tiny-unigram");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"Hello world", " Hello world ", "Hello world.\nSecond line",
+ "3.14159 x 42", "family emoji", " leading and trailing ", "a", "", " "})
+ void testAlignedMatchesUnaligned(String input) {
+ final AlignedText aligned = unigram().normalizeAligned(input);
+ assertEquals(unigram().normalize(input).toString(), aligned.normalizedString());
+ assertEquals(input, aligned.original().toString());
+ assertEquals(aligned.normalizedString().length(), aligned.alignment().normalizedLength());
+ assertEquals(input.length(), aligned.alignment().originalLength());
+ }
+
+ @Test
+ void testWordMapsBackThroughCollapsedWhitespace() {
+ final String input = " Hello world ";
+ final AlignedText aligned = unigram().normalizeAligned(input);
+ final String normalized = aligned.normalizedString();
+
+ final int at = normalized.indexOf("world");
+ final Span original = aligned.toOriginalSpan(at, at + "world".length());
+ assertEquals("world", input.substring(original.getStart(), original.getEnd()));
+ }
+
+ @Test
+ void testLigatureReplacementMapsToItsSourceCharacter() {
+ // The character map expands the single ligature to two letters; both normalized letters
+ // must map back to the one original character.
+ final String input = cp(0xFB01) + "nancial";
+ final AlignedText aligned = unigram().normalizeAligned(input);
+ final String normalized = aligned.normalizedString();
+
+ final int at = normalized.indexOf("fi");
+ assertTrue(at >= 0, "the character map must expand the ligature, got " + normalized);
+ final Span original = aligned.toOriginalSpan(at, at + 2);
+ assertEquals(0, original.getStart());
+ assertEquals(1, original.getEnd());
+ }
+
+ @Test
+ void testSupplementaryCharacterSpansUseUtf16Units() {
+ final String input = "I love " + new String(Character.toChars(0x1F355)) + " pizza";
+ final List pieces = unigram().encode(input);
+
+ SubwordPiece pizzaSlice = null;
+ for (final SubwordPiece piece : pieces) {
+ if (piece.piece().contains(new String(Character.toChars(0x1F355)))) {
+ pizzaSlice = piece;
+ }
+ }
+ assertTrue(pizzaSlice != null, "the emoji must surface as a piece, got " + pieces);
+ assertEquals(7, pizzaSlice.start());
+ assertEquals(9, pizzaSlice.end());
+ assertEquals(new String(Character.toChars(0x1F355)),
+ input.substring(pizzaSlice.start(), pizzaSlice.end()));
+ }
+
+ @Test
+ void testEverySpanIsWithinTheOriginalText() {
+ final String input = "quotes " + cp(0x201C) + "fancy" + cp(0x201D) + " and "
+ + cp(0x2018) + "single" + cp(0x2019) + " " + cp(0x2014) + " dash";
+ for (final SubwordPiece piece : unigram().encode(input)) {
+ assertTrue(piece.start() >= 0 && piece.end() <= input.length(),
+ "span " + piece + " must lie inside the input");
+ assertTrue(piece.start() <= piece.end(), "span " + piece + " must not be inverted");
+ }
+ }
+
+ @Test
+ void testSpansAreMonotonicAndAdjacent() {
+ final String input = "The quick brown fox jumps over the lazy dog.";
+ int previousEnd = 0;
+ for (final SubwordPiece piece : unigram().encode(input)) {
+ assertTrue(piece.start() >= previousEnd || piece.start() == piece.end(),
+ "piece " + piece + " must not step back before " + previousEnd);
+ previousEnd = Math.max(previousEnd, piece.end());
+ }
+ assertEquals(input.length(), previousEnd, "the last span must reach the end of the input");
+ }
+
+ @Test
+ void testUnpairedSurrogateIsDeterministic() {
+ final String input = "a" + (char) 0xD83C + "b";
+ final List first = unigram().encode(input);
+ final List second = unigram().encode(input);
+ assertEquals(first, second);
+ int covered = 0;
+ for (final SubwordPiece piece : first) {
+ covered = Math.max(covered, piece.end());
+ }
+ assertEquals(input.length(), covered);
+ }
+
+ private static String cp(int codePoint) {
+ return new String(Character.toChars(codePoint));
+ }
+
+ @Test
+ void testNullInputsFailLoudly() {
+ assertThrows(IllegalArgumentException.class, () -> unigram().encode(null));
+ assertThrows(IllegalArgumentException.class, () -> unigram().normalizeAligned(null));
+ assertThrows(IllegalArgumentException.class, () -> unigram().normalize(null));
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java
new file mode 100644
index 0000000000..a2c06d6177
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Stream;
+
+import opennlp.tools.tokenize.SubwordPiece;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+/**
+ * Shared support for the bundled test models and for the tab-separated parity fixture files
+ * produced by the {@code gen_fixtures.py} and {@code gen_real_fixtures.py} scripts in the test
+ * resources: one line per input, holding the input, the expected piece count, four columns per
+ * expected piece (content, id, start, end), and the expected normalized form.
+ */
+final class SentencePieceFixtures {
+
+ /** The file name suffix of a bundled or downloaded SentencePiece model. */
+ static final String MODEL_SUFFIX = ".model";
+
+ /** The file name suffix of the fixture file belonging to a model. */
+ static final String FIXTURES_SUFFIX = ".fixtures.tsv";
+
+ private static final Map LOADED = new ConcurrentHashMap<>();
+
+ private SentencePieceFixtures() {
+ }
+
+ /**
+ * The bundled models, one per algorithm and normalizer variant the reader must handle.
+ *
+ * @return The model names, usable as {@code @MethodSource} arguments.
+ */
+ static Stream models() {
+ return Stream.of("tiny-unigram", "tiny-unigram-bytefb", "tiny-bpe", "tiny-unigram-identity",
+ "tiny-unigram-suffix");
+ }
+
+ /**
+ * Loads a bundled model from the test resources, caching the result so the parsing cost is paid
+ * once per model across all test classes.
+ *
+ * @param model The bundled model name, without the {@link #MODEL_SUFFIX} suffix.
+ * @return The loaded tokenizer, shared by all callers.
+ */
+ static SentencePieceTokenizer tokenizer(String model) {
+ return LOADED.computeIfAbsent(model, name -> {
+ try (InputStream in =
+ SentencePieceFixtures.class.getResourceAsStream(name + MODEL_SUFFIX)) {
+ assertNotNull(in, "missing test resource " + name + MODEL_SUFFIX);
+ return SentencePieceTokenizer.load(in);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ });
+ }
+
+ /**
+ * Reads the fixture file belonging to a bundled model.
+ *
+ * @param model The bundled model name, without the {@link #MODEL_SUFFIX} suffix.
+ * @return The parsed fixtures in file order.
+ * @throws IOException Thrown if the fixture resource cannot be read.
+ */
+ static List fixtures(String model) throws IOException {
+ try (InputStream in =
+ SentencePieceFixtures.class.getResourceAsStream(model + FIXTURES_SUFFIX)) {
+ assertNotNull(in, "missing test resource " + model + FIXTURES_SUFFIX);
+ return read(new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)));
+ }
+ }
+
+ /**
+ * One parsed fixture line: an input with the piece sequence and normalized form the reference
+ * implementation produced for it.
+ *
+ * @param input The text to encode.
+ * @param pieces The expected pieces with ids and original-text spans, in text order.
+ * @param normalized The expected normalized form of {@code input}.
+ */
+ record Fixture(String input, List pieces, String normalized) {
+ }
+
+ /**
+ * Reads all fixture lines from a reader.
+ *
+ * @param reader The reader positioned at the start of a fixture file; must not be null.
+ * @return The parsed fixtures in file order.
+ * @throws IOException Thrown if the reader fails.
+ */
+ static List read(BufferedReader reader) throws IOException {
+ final List fixtures = new ArrayList<>();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ final String[] cols = line.split("\t", -1);
+ final String input = unescape(cols[0]);
+ final int count = Integer.parseInt(cols[1]);
+ final List pieces = new ArrayList<>(count);
+ for (int i = 0; i < count; i++) {
+ pieces.add(new SubwordPiece(unescape(cols[2 + i * 4]),
+ Integer.parseInt(cols[3 + i * 4]), Integer.parseInt(cols[4 + i * 4]),
+ Integer.parseInt(cols[5 + i * 4])));
+ }
+ fixtures.add(new Fixture(input, pieces, unescape(cols[2 + count * 4])));
+ }
+ return fixtures;
+ }
+
+ /**
+ * Asserts that a tokenizer reproduces one fixture exactly: the piece sequence with ids and
+ * spans, and the normalized form.
+ *
+ * @param tokenizer The tokenizer under test.
+ * @param fixture The expected encoding.
+ * @param context A prefix for failure messages that identifies the model and input.
+ */
+ static void assertFixture(SentencePieceTokenizer tokenizer, Fixture fixture, String context) {
+ final List actual = tokenizer.encode(fixture.input());
+ assertEquals(fixture.pieces().size(), actual.size(),
+ context + " piece count; got " + actual);
+ for (int i = 0; i < actual.size(); i++) {
+ final SubwordPiece expected = fixture.pieces().get(i);
+ final SubwordPiece got = actual.get(i);
+ assertEquals(expected.piece(), got.piece(), context + " piece " + i);
+ assertEquals(expected.id(), got.id(), context + " id of piece " + i);
+ assertEquals(expected.start(), got.start(), context + " start of piece " + i);
+ assertEquals(expected.end(), got.end(), context + " end of piece " + i);
+ }
+ assertEquals(fixture.normalized(), tokenizer.normalize(fixture.input()).toString(),
+ context + " normalized form");
+ }
+
+ /**
+ * Reverses the fixture files' escaping of tab, newline, carriage return, and backslash.
+ *
+ * @param s The escaped column content.
+ * @return The unescaped text.
+ * @throws IllegalArgumentException Thrown if an unknown escape sequence occurs.
+ */
+ static String unescape(String s) {
+ final StringBuilder out = new StringBuilder(s.length());
+ for (int i = 0; i < s.length(); i++) {
+ final char c = s.charAt(i);
+ if (c == '\\' && i + 1 < s.length()) {
+ i++;
+ switch (s.charAt(i)) {
+ case 't' -> out.append('\t');
+ case 'n' -> out.append('\n');
+ case 'r' -> out.append('\r');
+ case '\\' -> out.append('\\');
+ default -> throw new IllegalArgumentException("bad escape in fixture: " + s);
+ }
+ } else {
+ out.append(c);
+ }
+ }
+ return out.toString();
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java
new file mode 100644
index 0000000000..a207625290
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.tokenize.SubwordPiece;
+import opennlp.tools.util.InvalidFormatException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Fail-loud behavior on malformed models, plus the concurrency guarantee: one loaded tokenizer
+ * must produce identical results from many threads.
+ */
+class SentencePieceModelValidationTest {
+
+ @Test
+ void testNullAndEmptyInputFailLoudly() {
+ assertThrows(IllegalArgumentException.class,
+ () -> SentencePieceTokenizer.load((Path) null));
+ assertThrows(IllegalArgumentException.class,
+ () -> SentencePieceTokenizer.load((InputStream) null));
+ assertThrows(InvalidFormatException.class,
+ () -> SentencePieceTokenizer.load(new ByteArrayInputStream(new byte[0])));
+ }
+
+ @Test
+ void testGarbageBytesFailLoudly() {
+ final byte[] garbage = "this is not a model file at all".getBytes(StandardCharsets.UTF_8);
+ assertThrows(InvalidFormatException.class,
+ () -> SentencePieceTokenizer.load(new ByteArrayInputStream(garbage)));
+ }
+
+ @Test
+ void testTruncatedModelFailsLoudly() throws IOException {
+ final byte[] whole = readModel();
+ final byte[] truncated = Arrays.copyOf(whole, whole.length / 3);
+ assertThrows(InvalidFormatException.class,
+ () -> SentencePieceTokenizer.load(new ByteArrayInputStream(truncated)));
+ }
+
+ @Test
+ void testUnsupportedModelTypeFailsLoudly() {
+ // A minimal well-formed model claiming the WORD algorithm (model_type = 3).
+ final byte[] model = minimalModel(3);
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class,
+ () -> SentencePieceTokenizer.load(new ByteArrayInputStream(model)));
+ assertTrue(e.getMessage().contains("not supported"), e.getMessage());
+ }
+
+ @Test
+ void testMissingUnknownPieceFailsLoudly() {
+ final byte[] model = minimalModelWithoutUnk();
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class,
+ () -> SentencePieceTokenizer.load(new ByteArrayInputStream(model)));
+ assertTrue(e.getMessage().contains("unknown piece"), e.getMessage());
+ }
+
+ @Test
+ void testConcurrentEncodingIsConsistent() throws Exception {
+ final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer("tiny-unigram");
+ final String[] inputs = {
+ "The quick brown fox jumps over the lazy dog.",
+ "tokenization and segmentation",
+ " Hello world ",
+ "water running walked faster apple book work play"};
+ final List> expected = new ArrayList<>();
+ for (final String input : inputs) {
+ expected.add(tokenizer.encode(input));
+ }
+
+ final ExecutorService pool = Executors.newFixedThreadPool(8);
+ try {
+ final List> futures = new ArrayList<>();
+ for (int t = 0; t < 8; t++) {
+ futures.add(pool.submit((Callable) () -> {
+ for (int round = 0; round < 500; round++) {
+ for (int i = 0; i < inputs.length; i++) {
+ if (!expected.get(i).equals(tokenizer.encode(inputs[i]))) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }));
+ }
+ for (final Future future : futures) {
+ assertTrue(future.get(), "concurrent encoding must match single-threaded results");
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+
+ @Test
+ void testVocabularyAccessors() {
+ final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer("tiny-unigram");
+ assertEquals(300, tokenizer.vocabularySize());
+ assertEquals(SentencePieceTokenizer.Algorithm.UNIGRAM, tokenizer.algorithm());
+ for (int id = 0; id < tokenizer.vocabularySize(); id++) {
+ final String piece = tokenizer.idToPiece(id);
+ if (!tokenizer.isUnknown(id) && !tokenizer.isControl(id)) {
+ assertEquals(id, tokenizer.pieceToId(piece), "round trip of piece '" + piece + "'");
+ }
+ }
+ assertEquals(tokenizer.unknownId(), tokenizer.pieceToId("definitely-not-in-the-vocabulary"));
+ assertThrows(IllegalArgumentException.class, () -> tokenizer.idToPiece(-1));
+ assertThrows(IllegalArgumentException.class,
+ () -> tokenizer.idToPiece(tokenizer.vocabularySize()));
+ assertThrows(IllegalArgumentException.class, () -> tokenizer.pieceToId(null));
+ }
+
+ @Test
+ void testScoresAreFiniteAndRangeChecked() {
+ final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer("tiny-unigram");
+ for (int id = 0; id < tokenizer.vocabularySize(); id++) {
+ assertTrue(Float.isFinite(tokenizer.score(id)), "score of piece " + id);
+ }
+ assertThrows(IllegalArgumentException.class, () -> tokenizer.score(-1));
+ assertThrows(IllegalArgumentException.class,
+ () -> tokenizer.score(tokenizer.vocabularySize()));
+ }
+
+ @Test
+ void testBytePiecesExistOnlyInByteFallbackModels() {
+ final SentencePieceTokenizer byteFallback =
+ SentencePieceFixtures.tokenizer("tiny-unigram-bytefb");
+ int bytePieces = 0;
+ for (int id = 0; id < byteFallback.vocabularySize(); id++) {
+ if (byteFallback.isByte(id)) {
+ bytePieces++;
+ assertTrue(byteFallback.idToPiece(id).startsWith("<0x"),
+ "byte piece " + id + " is " + byteFallback.idToPiece(id));
+ }
+ }
+ assertEquals(256, bytePieces, "byte fallback defines one piece per byte value");
+
+ final SentencePieceTokenizer plain = SentencePieceFixtures.tokenizer("tiny-unigram");
+ for (int id = 0; id < plain.vocabularySize(); id++) {
+ assertFalse(plain.isByte(id), "piece " + id + " must not be a byte piece");
+ }
+ assertThrows(IllegalArgumentException.class, () -> plain.isByte(-1));
+ }
+
+ private static byte[] readModel() throws IOException {
+ try (InputStream in =
+ SentencePieceModelValidationTest.class.getResourceAsStream("tiny-unigram.model")) {
+ return in.readAllBytes();
+ }
+ }
+
+ // Hand-encodes a minimal ModelProto: three pieces (, , ) and a trainer spec with
+ // the requested model type.
+ private static byte[] minimalModel(int modelType) {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ writePiece(out, "", 2);
+ writePiece(out, "", 3);
+ writePiece(out, "", 3);
+ writePiece(out, "a", 1);
+ // trainer_spec { model_type = }
+ out.write(0x12);
+ out.write(2);
+ out.write(0x18);
+ out.write(modelType);
+ return out.toByteArray();
+ }
+
+ private static byte[] minimalModelWithoutUnk() {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ writePiece(out, "a", 1);
+ writePiece(out, "b", 1);
+ return out.toByteArray();
+ }
+
+ private static void writePiece(ByteArrayOutputStream out, String piece, int type) {
+ final byte[] utf8 = piece.getBytes(StandardCharsets.UTF_8);
+ // pieces { piece = ; score = 0.0; type = } as nested length-delimited field 1.
+ final int inner = 2 + utf8.length + 2;
+ out.write(0x0A);
+ out.write(inner);
+ out.write(0x0A);
+ out.write(utf8.length);
+ out.writeBytes(utf8);
+ out.write(0x18);
+ out.write(type);
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java
new file mode 100644
index 0000000000..f6361a93b2
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.StringJoiner;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Asserts exact parity with the reference implementation: for every fixture input, the pieces,
+ * ids, original-text spans, and the normalized form must equal what the reference produced for
+ * the same bundled model. The fixtures were generated by the {@code gen_fixtures.py} script in
+ * the test resources against the sentencepiece Python package.
+ */
+class SentencePieceParityTest {
+
+ @ParameterizedTest
+ @MethodSource("opennlp.subword.sentencepiece.SentencePieceFixtures#models")
+ void testFixtureParity(String model) throws IOException {
+ final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer(model);
+ int lines = 0;
+ for (final SentencePieceFixtures.Fixture fixture : SentencePieceFixtures.fixtures(model)) {
+ lines++;
+ SentencePieceFixtures.assertFixture(tokenizer, fixture,
+ model + " input <" + fixture.input() + ">");
+ }
+ assertTrue(lines >= 30, "the fixture file must not be empty or truncated");
+ }
+
+ @ParameterizedTest
+ @MethodSource("opennlp.subword.sentencepiece.SentencePieceFixtures#models")
+ void testEmbeddedSelfTestSamples(String model) {
+ final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer(model);
+ final List inputs = tokenizer.selfTestInputs();
+ final List expected = tokenizer.selfTestExpected();
+ assertTrue(!inputs.isEmpty(), "the tiny models embed self-test samples");
+ for (int i = 0; i < inputs.size(); i++) {
+ final StringJoiner joined = new StringJoiner(" ");
+ for (final String piece : tokenizer.encodeToPieces(inputs.get(i))) {
+ joined.add(piece);
+ }
+ assertEquals(expected.get(i), joined.toString(),
+ model + " self-test sample <" + inputs.get(i) + ">");
+ }
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java
new file mode 100644
index 0000000000..1174f8652c
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+/**
+ * Opt-in parity check against real pre-trained models, which are downloaded rather than bundled.
+ *
+ * Point {@code -Dopennlp.subword.eval.dir} at a directory holding {@code .model} files
+ * with sibling {@code .fixtures.tsv} files generated by the {@code gen_real_fixtures.py}
+ * script from the test resources; every model found is asserted piece for piece. Without the
+ * property the test is skipped.
+ */
+class SentencePieceRealModelEvalTest {
+
+ @Test
+ void testRealModelParity() throws IOException {
+ final String dir = System.getProperty("opennlp.subword.eval.dir");
+ assumeTrue(dir != null && !dir.isBlank(),
+ "set -Dopennlp.subword.eval.dir to run the real-model parity check");
+
+ int models = 0;
+ try (Stream files = Files.list(Path.of(dir))) {
+ for (final Path model : files
+ .filter(f -> f.toString().endsWith(SentencePieceFixtures.MODEL_SUFFIX))
+ .sorted().toList()) {
+ final String path = model.toString();
+ final Path fixtures = Path.of(
+ path.substring(0, path.length() - SentencePieceFixtures.MODEL_SUFFIX.length())
+ + SentencePieceFixtures.FIXTURES_SUFFIX);
+ assumeTrue(Files.exists(fixtures), "no fixtures for " + model.getFileName());
+ models++;
+ assertModel(model, fixtures);
+ }
+ }
+ assertTrue(models > 0, "the eval directory contains no models");
+ }
+
+ private static void assertModel(Path modelPath, Path fixturesPath) throws IOException {
+ final SentencePieceTokenizer tokenizer = SentencePieceTokenizer.load(modelPath);
+ final List fixtures;
+ try (BufferedReader reader = Files.newBufferedReader(fixturesPath, StandardCharsets.UTF_8)) {
+ fixtures = SentencePieceFixtures.read(reader);
+ }
+ for (final SentencePieceFixtures.Fixture fixture : fixtures) {
+ SentencePieceFixtures.assertFixture(tokenizer, fixture,
+ modelPath.getFileName() + " input <" + fixture.input() + ">");
+ }
+ assertTrue(fixtures.size() >= 30, modelPath.getFileName() + " fixtures must not be truncated");
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java
new file mode 100644
index 0000000000..e5bb42d148
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InvalidClassException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.ArrayList;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertIterableEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Asserts the {@code Serializable} contract inherited through
+ * {@code opennlp.tools.util.normalizer.CharSequenceNormalizer}: a tokenizer round-tripped
+ * through Java object serialization must encode and normalize exactly like the original.
+ * Also asserts the guarded read path of
+ * {@link SentencePieceTokenizer#deserialize(InputStream)}: foreign payloads and streams
+ * exceeding the resource limits are rejected before materialisation.
+ */
+class SentencePieceTokenizerSerializationTest {
+
+ private static final String[] INPUTS = {
+ "",
+ "The quick brown fox jumps over the lazy dog.",
+ " Hello world ",
+ "tokenization and segmentation",
+ "caf\u00e9 na\u00efve \u4e2d\u6587"
+ };
+
+ @ParameterizedTest
+ @MethodSource("opennlp.subword.sentencepiece.SentencePieceFixtures#models")
+ void testRoundTripPreservesEncoding(String model) throws IOException, ClassNotFoundException {
+ final SentencePieceTokenizer original = SentencePieceFixtures.tokenizer(model);
+
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ObjectOutputStream out = new ObjectOutputStream(bytes)) {
+ out.writeObject(original);
+ }
+ final SentencePieceTokenizer copy;
+ try (ObjectInputStream in =
+ new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) {
+ copy = (SentencePieceTokenizer) in.readObject();
+ }
+
+ assertEquals(original.algorithm(), copy.algorithm(), model + " algorithm");
+ assertEquals(original.vocabularySize(), copy.vocabularySize(), model + " vocabulary size");
+ for (final String input : INPUTS) {
+ final String context = model + " input <" + input + ">";
+ assertIterableEquals(original.encode(input), copy.encode(input), context + " pieces");
+ assertEquals(original.normalize(input).toString(), copy.normalize(input).toString(),
+ context + " normalized form");
+ }
+ }
+
+ /**
+ * Serializes the tokenizer of the given fixture model through
+ * {@link SentencePieceTokenizer#serialize(java.io.OutputStream)}.
+ *
+ * @param model The fixture model name.
+ * @return The serialized bytes.
+ * @throws IOException Thrown if the model cannot be read or serialized.
+ */
+ private static byte[] serialized(String model) throws IOException {
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ SentencePieceFixtures.tokenizer(model).serialize(bytes);
+ return bytes.toByteArray();
+ }
+
+ @ParameterizedTest
+ @MethodSource("opennlp.subword.sentencepiece.SentencePieceFixtures#models")
+ void testGuardedDeserializePreservesEncoding(String model)
+ throws IOException, ClassNotFoundException {
+ final SentencePieceTokenizer original = SentencePieceFixtures.tokenizer(model);
+ final SentencePieceTokenizer copy =
+ SentencePieceTokenizer.deserialize(new ByteArrayInputStream(serialized(model)));
+
+ assertEquals(original.algorithm(), copy.algorithm(), model + " algorithm");
+ for (final String input : INPUTS) {
+ final String context = model + " input <" + input + ">";
+ assertIterableEquals(original.encode(input), copy.encode(input), context + " pieces");
+ }
+ }
+
+ /**
+ * Verifies that a stream whose top-level object is not on the allow-list is rejected
+ * before it is materialised, even though its classes are harmless JDK types.
+ */
+ @Test
+ void testForeignPayloadIsRejected() throws IOException {
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ObjectOutputStream out = new ObjectOutputStream(bytes)) {
+ final ArrayList foreign = new ArrayList<>();
+ foreign.add("not a tokenizer");
+ out.writeObject(foreign);
+ }
+ assertThrows(InvalidClassException.class, () ->
+ SentencePieceTokenizer.deserialize(new ByteArrayInputStream(bytes.toByteArray())));
+ }
+
+ /**
+ * Verifies that a legitimate stream is rejected when it exceeds the supplied resource
+ * limits, so the limits bound the graph regardless of the class allow-list.
+ */
+ @Test
+ void testStreamExceedingLimitsIsRejected() throws IOException {
+ final byte[] legitimate = serialized("tiny-unigram");
+ final SentencePieceTokenizer.DeserializationLimits tight =
+ new SentencePieceTokenizer.DeserializationLimits(1, 1, 1);
+ assertThrows(InvalidClassException.class, () ->
+ SentencePieceTokenizer.deserialize(new ByteArrayInputStream(legitimate), tight));
+ }
+
+ /**
+ * Verifies that null arguments are rejected with {@link IllegalArgumentException} at the
+ * API boundary.
+ */
+ @Test
+ void testNullArgumentsAreRejected() throws IOException {
+ final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer("tiny-unigram");
+ assertThrows(IllegalArgumentException.class, () -> tokenizer.serialize(null));
+ assertThrows(IllegalArgumentException.class, () ->
+ SentencePieceTokenizer.deserialize(null));
+ assertThrows(IllegalArgumentException.class, () ->
+ SentencePieceTokenizer.deserialize(new ByteArrayInputStream(new byte[0]), null));
+ assertThrows(IllegalArgumentException.class, () ->
+ new SentencePieceTokenizer.DeserializationLimits(0, 1, 1));
+ assertThrows(IllegalArgumentException.class, () ->
+ new SentencePieceTokenizer.DeserializationLimits(1, 0, 1));
+ assertThrows(IllegalArgumentException.class, () ->
+ new SentencePieceTokenizer.DeserializationLimits(1, 1, 0));
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java
new file mode 100644
index 0000000000..33c375574b
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.subword.sentencepiece;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import opennlp.tools.tokenize.SubwordPiece;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Pins the cookbook path documented in {@code tokenizer.xml}: load a
+ * {@link SentencePieceTokenizer} from a {@code .model} file, encode text to pieces with
+ * original offsets, and obtain id arrays.
+ */
+public class SentencePieceUsageExampleTest {
+
+ @Test
+ void testLoadEncodeAndEncodeToIds(@TempDir Path dir) throws IOException {
+ final Path modelFile = dir.resolve("spiece.model");
+ try (InputStream in = SentencePieceUsageExampleTest.class
+ .getResourceAsStream("tiny-unigram.model")) {
+ assertNotNull(in, "missing test resource tiny-unigram.model");
+ Files.copy(in, modelFile);
+ }
+
+ final SentencePieceTokenizer tokenizer = SentencePieceTokenizer.load(modelFile);
+ final String text = "hello world";
+ final List pieces = tokenizer.encode(text);
+ assertFalse(pieces.isEmpty());
+ for (final SubwordPiece piece : pieces) {
+ assertTrue(piece.id() >= 0);
+ assertTrue(piece.start() >= 0);
+ assertTrue(piece.end() <= text.length());
+ // Control or whitespace pieces may report an empty span (start == end).
+ assertTrue(piece.start() <= piece.end());
+ }
+
+ final int[] ids = tokenizer.encodeToIds(text);
+ assertEquals(pieces.size(), ids.length);
+ for (int i = 0; i < ids.length; i++) {
+ assertEquals(pieces.get(i).id(), ids[i]);
+ }
+ }
+}
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/corpus.txt b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/corpus.txt
new file mode 100644
index 0000000000..d578654315
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/corpus.txt
@@ -0,0 +1,66 @@
+The quick brown fox jumps over the lazy dog.
+Apache OpenNLP is a machine learning based toolkit for the processing of natural language text.
+It supports the most common NLP tasks, such as tokenization, sentence segmentation, and named entity extraction.
+Subword tokenization decomposes words into smaller units drawn from a fixed vocabulary.
+The unigram language model selects the segmentation with the highest total log probability.
+Byte pair encoding merges the most frequent adjacent symbol pairs until no merge applies.
+A sentence piece model carries its own text normalizer inside the model file.
+Character offsets should always point back into the original text.
+Whitespace is escaped with a special marker so word boundaries survive segmentation.
+Numbers like 3.14159 and 42 and 1024 appear in ordinary text.
+Punctuation, quotes, and dashes are folded by the normalizer!
+Questions? Answers! Ellipses... and (parentheses) too.
+The cafe served naive patrons a souffle with creme fraiche.
+Internationalization and localization are long words.
+Antidisestablishmentarianism remains one of the longest English words.
+She sells seashells by the seashore.
+Peter Piper picked a peck of pickled peppers.
+How much wood would a woodchuck chuck if a woodchuck could chuck wood?
+The rain in Spain stays mainly in the plain.
+To be or not to be, that is the question.
+All happy families are alike; each unhappy family is unhappy in its own way.
+It was the best of times, it was the worst of times.
+Call me Ishmael.
+In the beginning was the word.
+The world is everything that is the case.
+Language models assign probabilities to sequences of tokens.
+Retrieval systems rank documents by similarity to a query.
+Embeddings map text into dense vector spaces.
+Search engines combine lexical and semantic signals.
+The tokenizer must be fast, deterministic, and thread safe.
+Model files are loaded once and shared across threads.
+Tests must prove parity with the reference implementation.
+Offsets are measured in code units of the original encoding.
+The normalizer collapses repeated whitespace into one marker.
+A leading marker separates words that start a sentence.
+Unknown characters fall back to a penalty score.
+Byte fallback decomposes unknown characters into byte pieces.
+User defined symbols are never split by the tokenizer.
+Control symbols never appear in encoded output.
+The vocabulary maps each piece to an integer identifier.
+Scores are log probabilities in the unigram model.
+Merge ranks order the byte pair encoding agenda.
+The trie enumerates every piece that starts at a position.
+Dynamic programming finds the best path in one pass.
+Backtracking recovers the winning segmentation.
+The agenda is a priority queue ordered by score.
+Stale entries are skipped when their symbols have merged.
+Un texto corto en espanol para variar el corpus.
+Un petit texte en francais pour la diversite.
+Ein kurzer deutscher Satz steht auch hier.
+Ancora una frase italiana per completezza.
+Tokenization quality depends on the training corpus.
+The model was trained on a tiny corpus for testing only.
+Nothing in this file is quoted from any external work.
+Short lines help.
+One.
+Two words.
+Three little words.
+water water water water water
+running runner ran runs
+walked walking walker walks
+faster fastest fast
+apple apples applesauce
+book books bookshelf bookstore
+work works worked working worker
+play plays played playing player
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_fixtures.py b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_fixtures.py
new file mode 100644
index 0000000000..658902e48f
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_fixtures.py
@@ -0,0 +1,139 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Trains the tiny SentencePiece test models and generates the parity fixtures.
+
+Run inside a venv with the sentencepiece package installed:
+ python gen_fixtures.py
+
+Every fixture line is tab-separated with backslash escaping (\\\\, \\t, \\n, \\r):
+ esc(input) TAB pieceCount TAB [esc(piece) TAB id TAB begin TAB end]... TAB esc(normalized)
+Offsets are UTF-16 code-unit offsets into the original input, matching Java string indexing.
+"""
+import sys
+import sentencepiece as spm
+
+MULTILINGUAL = [
+ "Le café coûte trois euros à Paris.",
+ "Der Straßenname ändert sich häufig.",
+ "Ça va très bien, merci beaucoup.",
+ "El niño pequeño come una manzana.",
+ "Привет мир и всем добро.",
+ "東京タワーに登りました。",
+ "日本語の文章も少しあります。",
+ "안녕하세요 세계입니다.",
+ "你好世界这是中文。",
+ "I love \U0001f355 and \U0001f1e9\U0001f1ea a lot!",
+ "Emoji test \U0001f600 \U0001f680 ❤️ done.",
+]
+
+INPUTS = [
+ "",
+ " ",
+ " ",
+ "a",
+ "Hello world",
+ " Hello world ",
+ "Hello world.\nSecond line\ttabbed",
+ "The quick brown fox jumps over the lazy dog.",
+ "tokenization and segmentation",
+ "Antidisestablishmentarianism",
+ "water running walked faster apple book work play",
+ "3.14159 x 42 = 1024?",
+ "!!!???...",
+ "(parentheses) and [brackets] and {braces}",
+ "café naïve fiancé résumé",
+ "financial fluid",
+ "① ⑪ ㋿ ＫＡＴＡＫＡＮＡ",
+ "ｶﾀｶﾅ half width",
+ "東京タワーへ行きました",
+ "日本語とEnglish混在",
+ "Привет мир",
+ "안녕하세요 세계",
+ "你好，世界！",
+ "I love \U0001f355 pizza",
+ "flags \U0001f1e9\U0001f1ea \U0001f1fa\U0001f1f8 end",
+ "family \U0001f469\U0001f469\U0001f467\U0001f466 emoji",
+ "zerowidth and non breaking",
+ "quotes “fancy” and ‘single’ — dash",
+ " the [URL] token",
+ "a b[URL]c",
+ "control tokens inline",
+ "https://example.com/path?q=1&x=2",
+ "UPPER lower MiXeD case",
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "Ω≈ç√∫˜µ≤",
+ "مرحبا بالعالم",
+ " leading and trailing ",
+ "\ttab\tstart",
+ "newline\n\n\nruns",
+ "mid spaces collapse",
+]
+
+MODELS = {
+ "tiny-unigram": dict(model_type="unigram", vocab_size=300),
+ "tiny-unigram-bytefb": dict(model_type="unigram", vocab_size=600, byte_fallback=True,
+ character_coverage=0.995),
+ "tiny-bpe": dict(model_type="bpe", vocab_size=300),
+ "tiny-unigram-identity": dict(model_type="unigram", vocab_size=300,
+ normalization_rule_name="identity"),
+ "tiny-unigram-suffix": dict(model_type="unigram", vocab_size=300,
+ treat_whitespace_as_suffix=True),
+}
+
+
+def esc(s):
+ return (s.replace("\\", "\\\\").replace("\t", "\\t")
+ .replace("\n", "\\n").replace("\r", "\\r"))
+
+
+def utf16_offset(text, codepoint_offset):
+ return len(text[:codepoint_offset].encode("utf-16-le")) // 2
+
+
+def main(corpus, outdir):
+ full_corpus = outdir + "/corpus-full.txt"
+ with open(corpus, encoding="utf-8") as f:
+ lines = f.read().splitlines()
+ lines += MULTILINGUAL
+ with open(full_corpus, "w", encoding="utf-8") as f:
+ f.write("\n".join(lines) + "\n")
+
+ for name, opts in MODELS.items():
+ spm.SentencePieceTrainer.Train(
+ input=full_corpus,
+ model_prefix=outdir + "/" + name,
+ hard_vocab_limit=False,
+ character_coverage=opts.pop("character_coverage", 1.0),
+ user_defined_symbols=["", "[URL]"],
+ self_test_sample_size=10,
+ **opts,
+ )
+ sp = spm.SentencePieceProcessor(model_file=outdir + "/" + name + ".model")
+ with open(outdir + "/" + name + ".fixtures.tsv", "w", encoding="utf-8") as out:
+ for text in INPUTS:
+ proto = sp.EncodeAsImmutableProto(text)
+ cols = [esc(text), str(len(proto.pieces))]
+ for piece in proto.pieces:
+ cols += [esc(piece.piece), str(piece.id),
+ str(utf16_offset(text, piece.begin)),
+ str(utf16_offset(text, piece.end))]
+ cols.append(esc(sp.Normalize(text)))
+ out.write("\t".join(cols) + "\n")
+ print(name, "vocab", sp.GetPieceSize())
+
+
+if __name__ == "__main__":
+ main(sys.argv[1], sys.argv[2])
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_real_fixtures.py b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_real_fixtures.py
new file mode 100644
index 0000000000..9b27c7a11e
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_real_fixtures.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generates parity fixtures for pre-trained real-world models (no training).
+
+Usage: python gen_real_fixtures.py
+Reads every *.model in the directory and writes a sibling *.fixtures.tsv in the same
+escaped-TSV format as gen_fixtures.py, over a larger and messier input list.
+"""
+import glob
+import os
+import sys
+import sentencepiece as spm
+from gen_fixtures import INPUTS, esc, utf16_offset
+
+EXTRA = [
+ "The Transformer architecture revolutionized natural language processing in 2017.",
+ "supercalifragilisticexpialidocious and pneumonoultramicroscopicsilicovolcanoconiosis",
+ "e=mc^2, F=ma, and a^2+b^2=c^2 are famous equations.",
+ "Mixed scripts: English, 日本語, 한국어, русский, and العربية together.",
+ "Prices: $19.99, €25,50, £12, ¥1500, and ₹999.",
+ "C++ and C# and F# are programming languages; so is Java.",
+ "def encode(text): return sp.encode(text, out_type=str)",
+ "SELECT * FROM documents WHERE score > 0.5 ORDER BY rank;",
+ "The 2024 Summer Olympics were held in Paris, France.",
+ "COVID-19 vaccines use mRNA technology (Pfizer-BioNTech, Moderna).",
+ "Email me at test.user+tag@example.co.uk or call +1 (555) 010-9999.",
+ "10,000 steps a day keeps the doctor away... allegedly!",
+ "The naive resume of the fiancee included a cafe visit.",
+ "¿Dónde está la biblioteca? ¡Allí está!",
+ "Smørrebrød og æbleskiver er danske specialiteter.",
+ "Zażółć gęślą jaźń is a Polish pangram.",
+ "Đây là tiếng Việt với nhiều dấu.",
+ "今日はいい天気ですね。明日も晴れるといいな。",
+ "北京和上海都是大城市。",
+ "한국의 수도는 서울입니다.",
+ "\U0001f9d1\U0001f4bb codes while \U0001f9d1\U0001f373 cooks \U0001f35c!",
+ "
line separator and
paragraph separator lurk here",
+ "BOM at the start of this sentence",
+ "tabs\tand\ttabs\tand\ttabs",
+ "CRLF\r\nline endings\r\nhappen",
+]
+
+
+def main(model_dir):
+ for model_path in sorted(glob.glob(os.path.join(model_dir, "*.model"))):
+ name = os.path.splitext(model_path)[0]
+ sp = spm.SentencePieceProcessor(model_file=model_path)
+ with open(name + ".fixtures.tsv", "w", encoding="utf-8") as out:
+ for text in INPUTS + EXTRA:
+ proto = sp.EncodeAsImmutableProto(text)
+ cols = [esc(text), str(len(proto.pieces))]
+ for piece in proto.pieces:
+ cols += [esc(piece.piece), str(piece.id),
+ str(utf16_offset(text, piece.begin)),
+ str(utf16_offset(text, piece.end))]
+ cols.append(esc(sp.Normalize(text)))
+ out.write("\t".join(cols) + "\n")
+ print(os.path.basename(name), "vocab", sp.GetPieceSize())
+
+
+if __name__ == "__main__":
+ main(sys.argv[1])
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.fixtures.tsv b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.fixtures.tsv
new file mode 100644
index 0000000000..e3560148f9
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.fixtures.tsv
@@ -0,0 +1,40 @@
+ 0
+ 0
+ 0
+a 1 ▁a 8 0 1 ▁a
+Hello world 7 ▁ 177 0 0 H 246 0 1 el 48 1 3 l 186 3 4 o 183 4 5 ▁wor 38 5 9 ld 129 9 11 ▁Hello▁world
+ Hello world 7 ▁ 177 1 1 H 246 1 2 el 48 2 4 l 186 4 5 o 183 5 6 ▁wor 38 6 12 ld 129 12 14 ▁Hello▁world
+Hello world.\nSecond line\ttabbed 19 ▁ 177 0 0 H 246 0 1 el 48 1 3 l 186 3 4 o 183 4 5 ▁wor 38 5 9 ld 129 9 11 . 193 11 12 ▁S 64 12 14 ec 51 14 16 on 16 16 18 d 189 18 19 ▁l 28 19 21 in 7 21 23 e 178 23 24 ▁t 5 24 26 ab 85 26 28 b 199 28 29 ed 24 29 31 ▁Hello▁world.▁Second▁line▁tabbed
+The quick brown fox jumps over the lazy dog. 27 ▁The 50 0 3 ▁qu 81 3 6 ic 61 6 8 k 196 8 9 ▁b 23 9 11 ro 41 11 13 wn 90 13 15 ▁f 25 15 17 o 183 17 18 x 203 18 19 ▁ 177 19 20 j 220 20 21 u 192 21 22 mp 108 22 24 s 182 24 25 ▁o 45 25 27 v 200 27 28 er 6 28 30 ▁the 19 30 34 ▁l 28 34 36 a 179 36 37 z 202 37 38 y 198 38 39 ▁d 42 39 41 o 183 41 42 g 194 42 43 . 193 43 44 ▁The▁quick▁brown▁fox▁jumps▁over▁the▁lazy▁dog.
+tokenization and segmentation 4 ▁tokeniz 170 0 7 ation 47 7 12 ▁and 59 12 16 ▁segmentation 171 16 29 ▁tokenization▁and▁segmentation
+Antidisestablishmentarianism 16 ▁A 76 0 1 n 181 1 2 t 180 2 3 id 176 3 5 is 31 5 7 est 57 7 10 ab 85 10 12 l 186 12 13 is 31 13 15 h 187 15 16 ment 83 16 20 ar 18 20 22 i 184 22 23 an 40 23 25 is 31 25 27 m 191 27 28 ▁Antidisestablishmentarianism
+water running walked faster apple book work play 15 ▁water 124 0 5 ▁r 93 5 7 un 39 7 9 ning 154 9 13 ▁wal 160 13 17 k 196 17 18 ed 24 18 20 ▁fast 162 20 25 er 6 25 27 ▁app 99 27 31 le 107 31 33 ▁boo 155 33 37 k 196 37 38 ▁work 167 38 43 ▁play 123 43 48 ▁water▁running▁walked▁faster▁apple▁book▁work▁play
+3.14159 x 42 = 1024? 21 ▁ 177 0 0 3 242 0 1 . 193 1 2 1 215 2 3 4 216 3 4 1 215 4 5 5 243 5 6 9 244 6 7 ▁ 177 7 8 x 203 8 9 ▁ 177 9 10 4 216 10 11 2 224 11 12 ▁ 177 12 13 = 0 13 14 ▁ 177 14 15 1 215 15 16 0 241 16 17 2 224 17 18 4 216 18 19 ? 225 19 20 ▁3.14159▁x▁42▁=▁1024?
+!!!???... 10 ▁ 177 0 0 ! 214 0 1 ! 214 1 2 ! 214 2 3 ? 225 3 4 ? 225 4 5 ? 225 5 6 . 193 6 7 . 193 7 8 . 193 8 9 ▁!!!???...
+(parentheses) and [brackets] and {braces} 25 ▁ 177 0 0 ( 239 0 1 p 190 1 2 ar 18 2 4 ent 35 4 7 he 9 7 9 ses 114 9 12 ) 240 12 13 ▁and 59 13 17 ▁ 177 17 18 [ 0 18 19 b 199 19 20 r 185 20 21 ack 112 21 24 et 86 24 26 s 182 26 27 ] 0 27 28 ▁and 59 28 32 ▁ 177 32 33 { 0 33 34 b 199 34 35 r 185 35 36 ac 29 36 38 es 11 38 40 } 0 40 41 ▁(parentheses)▁and▁[brackets]▁and▁{braces}
+café naïve fiancé résumé 18 ▁c 26 0 1 af 173 1 3 é 254 3 4 ▁n 49 4 6 a 179 6 7 ï 0 7 8 ve 109 8 10 ▁f 25 10 12 i 184 12 13 an 40 13 15 c 188 15 16 é 254 16 17 ▁r 93 17 19 é 254 19 20 s 182 20 21 u 192 21 22 m 191 22 23 é 254 23 24 ▁café▁naïve▁fiancé▁résumé
+financial fluid 10 ▁f 25 0 0 in 7 0 2 an 40 2 4 c 188 4 5 i 184 5 6 al 20 6 8 ▁f 25 8 9 l 186 9 10 u 192 10 11 id 176 11 13 ▁financial▁fluid
+① ⑪ ㋿ ＫＡＴＡＫＡＮＡ 16 ▁ 177 0 0 1 215 0 1 ▁ 177 1 2 1 215 2 2 1 215 2 3 ▁ 177 3 4 令和 0 4 5 ▁ 177 5 6 K 0 6 7 A 207 7 8 T 201 8 9 A 207 9 10 K 0 10 11 A 207 11 12 N 212 12 13 A 207 13 14 ▁1▁11▁令和▁KATAKANA
+ｶﾀｶﾅ half width 11 ▁ 177 0 0 カ 0 0 1 タ 268 1 2 カナ 0 2 4 ▁h 110 4 6 al 20 6 8 f 197 8 9 ▁w 14 9 11 id 176 11 13 t 180 13 14 h 187 14 15 ▁カタカナ▁half▁width
+東京タワーへ行きました 10 ▁ 177 0 0 東 280 0 1 京 273 1 2 タ 268 2 3 ワ 269 3 4 ー 270 4 5 へ行き 0 5 8 ま 235 8 9 し 234 9 10 た 264 10 11 ▁東京タワーへ行きました
+日本語とEnglish混在 12 ▁ 177 0 0 日 277 0 1 本 279 1 2 語 284 2 3 と 0 3 4 E 208 4 5 n 181 5 6 g 194 6 7 l 186 7 8 is 31 8 10 h 187 10 11 混在 0 11 13 ▁日本語とEnglish混在
+Привет мир 11 ▁ 177 0 0 П 256 0 1 р 222 1 2 и 221 2 3 в 230 3 4 е 231 4 5 т 260 5 6 ▁ 177 6 7 м 232 7 8 и 221 8 9 р 222 9 10 ▁Привет▁мир
+안녕하세요 세계 9 ▁ 177 0 0 안 290 0 1 녕 287 1 2 하 293 2 3 세 238 3 4 요 291 4 5 ▁ 177 5 6 세 238 6 7 계 286 7 8 ▁안녕하세요▁세계
+你好，世界！ 7 ▁ 177 0 0 你 274 0 1 好 275 1 2 , 204 2 3 世 271 3 4 界 281 4 5 ! 214 5 6 ▁你好,世界!
+I love 🍕 pizza 9 ▁I 92 0 1 ▁lo 96 1 4 ve 109 4 6 ▁ 177 6 7 🍕 297 7 9 ▁p 15 9 11 iz 54 11 13 z 202 13 14 a 179 14 15 ▁I▁love▁🍕▁pizza
+flags 🇩🇪 🇺🇸 end 12 ▁f 25 0 1 l 186 1 2 a 179 2 3 g 194 3 4 s 182 4 5 ▁ 177 5 6 🇩 295 6 8 🇪 296 8 10 ▁ 177 10 11 🇺🇸 0 11 15 ▁en 149 15 18 d 189 18 19 ▁flags▁🇩🇪▁🇺🇸▁end
+family 👩👩👧👦 emoji 12 ▁f 25 0 1 am 104 1 3 il 53 3 5 y 198 5 6 ▁ 177 6 7 👩👩👧👦 0 7 18 ▁ 177 18 19 e 178 19 20 m 191 20 21 o 183 21 22 j 220 22 23 i 184 23 24 ▁family▁👩👩👧👦▁emoji
+zerowidth and non breaking 16 ▁ 177 0 0 z 202 0 1 er 6 1 3 o 183 3 4 ▁w 14 4 6 id 176 6 8 t 180 8 9 h 187 9 10 ▁and 59 10 14 ▁n 49 14 16 on 16 16 18 ▁b 23 18 20 re 32 20 22 a 179 22 23 k 196 23 24 ing 27 24 27 ▁zero▁width▁and▁non▁breaking
+quotes “fancy” and ‘single’ — dash 22 ▁qu 81 0 2 ot 130 2 4 es 11 4 6 ▁ 177 6 7 “ 0 7 8 f 197 8 9 an 40 9 11 c 188 11 12 y 198 12 13 ” 0 13 14 ▁and 59 14 18 ▁ 177 18 19 ‘ 0 19 20 s 182 20 21 ing 27 21 24 le 107 24 26 ’ 0 26 27 ▁ 177 27 28 — 0 28 29 ▁d 42 29 31 as 30 31 33 h 187 33 34 ▁quotes▁“fancy”▁and▁‘single’▁—▁dash
+ the [URL] token 7 ▁ 177 0 0 3 0 6 ▁the 19 6 10 ▁ 177 10 11 [URL] 4 11 16 ▁to 43 16 19 ken 95 19 22 ▁▁the▁[URL]▁token
+a b[URL]c 6 ▁a 8 0 1 ▁ 177 1 2 3 2 8 b 199 8 9 [URL] 4 9 14 c 188 14 15 ▁a▁b[URL]c
+control tokens inline 20 ▁c 26 0 1 on 16 1 3 t 180 3 4 ro 41 4 6 l 186 6 7 ▁ 177 7 8 < 0 8 9 s 182 9 10 > 0 10 11 ▁to 43 11 14 ken 95 14 17 s 182 17 18 ▁ 177 18 19 0 22 23 ▁in 37 23 26 l 186 26 27 in 7 27 29 e 178 29 30 ▁control▁▁tokens▁▁inline
+https://example.com/path?q=1&x=2 26 ▁h 110 0 1 t 180 1 2 t 180 2 3 p 190 3 4 s 182 4 5 :// 0 5 8 ex 52 8 10 am 104 10 12 p 190 12 13 le 107 13 15 . 193 15 16 c 188 16 17 o 183 17 18 m 191 18 19 / 0 19 20 p 190 20 21 at 17 21 23 h 187 23 24 ? 225 24 25 q 205 25 26 = 0 26 27 1 215 27 28 & 0 28 29 x 203 29 30 = 0 30 31 2 224 31 32 ▁https://example.com/path?q=1&x=2
+UPPER lower MiXeD case 17 ▁U 135 0 1 P 210 1 2 P 210 2 3 E 208 3 4 R 248 4 5 ▁lo 96 5 8 w 195 8 9 er 6 9 11 ▁ 177 11 12 M 227 12 13 i 184 13 14 X 0 14 15 e 178 15 16 D 226 16 17 ▁c 26 17 19 as 30 19 21 e 178 21 22 ▁UPPER▁lower▁MiXeD▁case
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 40 ▁a 8 0 1 a 179 1 2 a 179 2 3 a 179 3 4 a 179 4 5 a 179 5 6 a 179 6 7 a 179 7 8 a 179 8 9 a 179 9 10 a 179 10 11 a 179 11 12 a 179 12 13 a 179 13 14 a 179 14 15 a 179 15 16 a 179 16 17 a 179 17 18 a 179 18 19 a 179 19 20 a 179 20 21 a 179 21 22 a 179 22 23 a 179 23 24 a 179 24 25 a 179 25 26 a 179 26 27 a 179 27 28 a 179 28 29 a 179 29 30 a 179 30 31 a 179 31 32 a 179 32 33 a 179 33 34 a 179 34 35 a 179 35 36 a 179 36 37 a 179 37 38 a 179 38 39 a 179 39 40 ▁aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+Ω≈ç√∫˜µ≤ 4 ▁ 177 0 0 Ω≈ç√∫ 0 0 5 ▁ 177 5 5 ̃μ≤ 0 5 8 ▁Ω≈ç√∫▁̃μ≤
+مرحبا بالعالم 4 ▁ 177 0 0 مرحبا 0 0 5 ▁ 177 5 6 بالعالم 0 6 13 ▁مرحبا▁بالعالم
+ leading and trailing 9 ▁l 28 2 3 e 178 3 4 ad 172 4 6 ing 27 6 9 ▁and 59 9 13 ▁t 5 13 15 ra 62 15 17 il 53 17 19 ing 27 19 22 ▁leading▁and▁trailing
+\ttab\tstart 6 ▁t 5 1 2 ab 85 2 4 ▁s 13 4 6 t 180 6 7 ar 18 7 9 t 180 9 10 ▁tab▁start
+newline\n\n\nruns 9 ▁n 49 0 1 e 178 1 2 w 195 2 3 l 186 3 4 in 7 4 6 e 178 6 7 ▁r 93 7 11 un 39 11 13 s 182 13 14 ▁newline▁runs
+mid spaces collapse 11 ▁m 22 0 1 id 176 1 3 ▁s 13 3 7 pac 147 7 10 es 11 10 12 ▁c 26 12 16 ol 69 16 18 l 186 18 19 ap 127 19 21 s 182 21 22 e 178 22 23 ▁mid▁spaces▁collapse
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.model b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.model
new file mode 100644
index 0000000000..8b6f22eb82
Binary files /dev/null and b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.model differ
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.fixtures.tsv b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.fixtures.tsv
new file mode 100644
index 0000000000..b8f9733dba
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.fixtures.tsv
@@ -0,0 +1,40 @@
+ 0
+ 0
+ 0
+a 1 ▁a 266 0 1 ▁a
+Hello world 7 ▁ 261 0 0 H 552 0 1 e 264 1 2 ll 412 2 4 o 274 4 5 ▁wor 427 5 9 ld 317 9 11 ▁Hello▁world
+ Hello world 7 ▁ 261 1 1 H 552 1 2 e 264 2 3 ll 412 3 5 o 274 5 6 ▁wor 427 6 12 ld 317 12 14 ▁Hello▁world
+Hello world.\nSecond line\ttabbed 21 ▁ 261 0 0 H 552 0 1 e 264 1 2 ll 412 2 4 o 274 4 5 ▁wor 427 5 9 ld 317 9 11 . 262 11 12 ▁S 296 12 14 e 264 14 15 c 411 15 16 o 274 16 17 nd 284 17 19 ▁l 492 19 21 ine 415 21 24 ▁ 261 24 25 t 269 25 26 a 279 26 27 b 344 27 28 b 344 28 29 ed 267 29 31 ▁Hello▁world.▁Second▁line▁tabbed
+The quick brown fox jumps over the lazy dog. 26 ▁Th 277 0 2 e 264 2 3 ▁qu 346 3 6 i 282 6 7 ck 307 7 9 ▁b 309 9 11 r 288 11 12 ow 306 12 14 n 268 14 15 ▁fo 359 15 18 x 421 18 19 ▁ 261 19 20 j 381 20 21 um 529 21 23 p 352 23 24 s 263 24 25 ▁o 513 25 27 ve 366 27 29 r 288 29 30 ▁the 265 30 34 ▁la 339 34 37 z 351 37 38 y 275 38 39 ▁do 464 39 42 g 356 42 43 . 262 43 44 ▁The▁quick▁brown▁fox▁jumps▁over▁the▁lazy▁dog.
+tokenization and segmentation 4 ▁tokenization 461 0 12 ▁a 266 12 14 nd 284 14 16 ▁segmentation 335 16 29 ▁tokenization▁and▁segmentation
+Antidisestablishmentarianism 15 ▁An 408 0 2 ti 523 2 4 d 273 4 5 is 278 5 7 est 323 7 10 a 279 10 11 b 344 11 12 lish 454 12 16 ment 508 16 20 ar 353 20 22 i 282 22 23 a 279 23 24 n 268 24 25 is 278 25 27 m 320 27 28 ▁Antidisestablishmentarianism
+water running walked faster apple book work play 15 ▁water 312 0 5 ▁runn 467 5 10 ing 272 10 13 ▁walk 348 13 18 ed 267 18 20 ▁fast 358 20 25 er 270 25 27 ▁app 498 27 31 le 405 31 33 ▁b 309 33 35 o 274 35 36 o 274 36 37 k 354 37 38 ▁work 303 38 43 ▁play 313 43 48 ▁water▁running▁walked▁faster▁apple▁book▁work▁play
+3.14159 x 42 = 1024? 21 ▁ 261 0 0 3 543 0 1 . 262 1 2 1 371 2 3 4 372 3 4 1 371 4 5 5 549 5 6 9 550 6 7 ▁ 261 7 8 x 421 8 9 ▁ 261 9 10 4 372 10 11 2 434 11 12 ▁ 261 12 13 <0x3D> 66 13 14 ▁ 261 14 15 1 371 15 16 0 548 16 17 2 434 17 18 4 372 18 19 ? 435 19 20 ▁3.14159▁x▁42▁=▁1024?
+!!!???... 10 ▁ 261 0 0 ! 370 0 1 ! 370 1 2 ! 370 2 3 ? 435 3 4 ? 435 4 5 ? 435 5 6 . 262 6 7 . 262 7 8 . 262 8 9 ▁!!!???...
+(parentheses) and [brackets] and {braces} 27 ▁ 261 0 0 ( 546 0 1 p 352 1 2 are 286 2 5 n 268 5 6 th 289 6 8 e 264 8 9 ses 414 9 12 ) 547 12 13 ▁a 266 13 15 nd 284 15 17 ▁ 261 17 18 <0x5B> 96 18 19 b 344 19 20 r 288 20 21 ack 327 21 24 e 264 24 25 ts 311 25 27 <0x5D> 98 27 28 ▁a 266 28 30 nd 284 30 32 ▁ 261 32 33 <0x7B> 128 33 34 b 344 34 35 ra 409 35 37 ces 478 37 40 <0x7D> 130 40 41 ▁(parentheses)▁and▁[brackets]▁and▁{braces}
+café naïve fiancé résumé 19 ▁caf 484 0 3 é 557 3 4 ▁ 261 4 5 n 268 5 6 a 279 6 7 <0xC3> 200 7 7 <0xAF> 180 7 8 ve 366 8 10 ▁fi 518 10 13 a 279 13 14 n 268 14 15 c 411 15 16 é 557 16 17 ▁ 261 17 18 r 288 18 19 é 557 19 20 s 263 20 21 um 529 21 23 é 557 23 24 ▁café▁naïve▁fiancé▁résumé
+financial fluid 13 ▁fi 518 0 1 n 268 1 2 a 279 2 3 n 268 3 4 c 411 4 5 i 282 5 6 al 281 6 8 ▁ 261 8 9 f 337 9 9 l 319 9 10 u 271 10 11 i 282 11 12 d 273 12 13 ▁financial▁fluid
+① ⑪ ㋿ ＫＡＴＡＫＡＮＡ 21 ▁ 261 0 0 1 371 0 1 ▁ 261 1 2 1 371 2 2 1 371 2 3 ▁ 261 3 4 <0xE4> 233 4 4 <0xBB> 192 4 4 <0xA4> 169 4 4 <0xE5> 234 4 4 <0x92> 151 4 4 <0x8C> 145 4 5 ▁ 261 5 6 <0x4B> 80 6 7 A 596 7 8 T 599 8 9 A 596 9 10 <0x4B> 80 10 11 A 596 11 12 N 542 12 13 A 596 13 14 ▁1▁11▁令和▁KATAKANA
+ｶﾀｶﾅ half width 17 ▁ 261 0 0 <0xE3> 232 0 0 <0x82> 135 0 0 <0xAB> 176 0 1 タ 565 1 2 <0xE3> 232 2 2 <0x82> 135 2 2 <0xAB> 176 2 3 <0xE3> 232 3 3 <0x83> 136 3 3 <0x8A> 143 3 4 ▁h 512 4 6 al 281 6 8 f 337 8 9 ▁wi 314 9 12 d 273 12 13 th 289 13 15 ▁カタカナ▁half▁width
+東京タワーへ行きました 18 ▁ 261 0 0 東 571 0 1 京 568 1 2 タ 565 2 3 ワ 580 3 4 ー 581 4 5 <0xE3> 232 5 5 <0x81> 134 5 5 <0xB8> 189 5 6 <0xE8> 237 6 6 <0xA1> 166 6 6 <0x8C> 145 6 7 <0xE3> 232 7 7 <0x81> 134 7 7 <0x8D> 146 7 8 ま 587 8 9 し 450 9 10 た 564 10 11 ▁東京タワーへ行きました
+日本語とEnglish混在 18 ▁ 261 0 0 日 583 0 1 本 585 1 2 <0xE8> 237 2 2 <0xAA> 175 2 2 <0x9E> 163 2 3 <0xE3> 232 3 3 <0x81> 134 3 3 <0xA8> 173 3 4 E 595 4 5 ng 328 5 7 lish 454 7 11 <0xE6> 235 11 11 <0xB7> 188 11 11 <0xB7> 188 11 12 <0xE5> 234 12 12 <0x9C> 161 12 12 <0xA8> 173 12 13 ▁日本語とEnglish混在
+Привет мир 11 ▁ 261 0 0 П 559 0 1 р 382 1 2 и 373 2 3 в 437 3 4 е 438 4 5 т 561 5 6 ▁ 261 6 7 м 439 7 8 и 373 8 9 р 382 9 10 ▁Привет▁мир
+안녕하세요 세계 19 ▁ 261 0 0 <0xEC> 241 0 0 <0x95> 154 0 0 <0x88> 141 0 1 <0xEB> 240 1 1 <0x85> 138 1 1 <0x95> 154 1 2 <0xED> 242 2 2 <0x95> 154 2 2 <0x98> 157 2 3 세 432 3 4 <0xEC> 241 4 4 <0x9A> 159 4 4 <0x94> 153 4 5 ▁ 261 5 6 세 432 6 7 <0xEA> 239 7 7 <0xB3> 184 7 7 <0x84> 137 7 8 ▁안녕하세요▁세계
+你好，世界！ 7 ▁ 261 0 0 你 569 0 1 好 570 1 2 , 280 2 3 世 566 3 4 界 572 4 5 ! 370 5 6 ▁你好,世界!
+I love 🍕 pizza 12 ▁I 329 0 1 ▁lo 305 1 4 ve 366 4 6 ▁ 261 6 7 <0xF0> 245 7 7 <0x9F> 164 7 7 <0x8D> 146 7 7 <0x95> 154 7 9 ▁p 486 9 11 i 282 11 12 z 351 12 13 za 540 13 15 ▁I▁love▁🍕▁pizza
+flags 🇩🇪 🇺🇸 end 26 ▁ 261 0 0 f 337 0 1 l 319 1 2 a 279 2 3 g 356 3 4 s 263 4 5 ▁ 261 5 6 <0xF0> 245 6 6 <0x9F> 164 6 6 <0x87> 140 6 6 <0xA9> 174 6 8 <0xF0> 245 8 8 <0x9F> 164 8 8 <0x87> 140 8 8 <0xAA> 175 8 10 ▁ 261 10 11 <0xF0> 245 11 11 <0x9F> 164 11 11 <0x87> 140 11 11 <0xBA> 191 11 13 <0xF0> 245 13 13 <0x9F> 164 13 13 <0x87> 140 13 13 <0xB8> 189 13 15 ▁en 318 15 18 d 273 18 19 ▁flags▁🇩🇪▁🇺🇸▁end
+family 👩👩👧👦 emoji 34 ▁famil 446 0 5 y 275 5 6 ▁ 261 6 7 <0xF0> 245 7 7 <0x9F> 164 7 7 <0x91> 150 7 7 <0xA9> 174 7 9 <0xE2> 231 9 9 <0x80> 133 9 9 <0x8D> 146 9 10 <0xF0> 245 10 10 <0x9F> 164 10 10 <0x91> 150 10 10 <0xA9> 174 10 12 <0xE2> 231 12 12 <0x80> 133 12 12 <0x8D> 146 12 13 <0xF0> 245 13 13 <0x9F> 164 13 13 <0x91> 150 13 13 <0xA7> 172 13 15 <0xE2> 231 15 15 <0x80> 133 15 15 <0x8D> 146 15 16 <0xF0> 245 16 16 <0x9F> 164 16 16 <0x91> 150 16 16 <0xA6> 171 16 18 ▁ 261 18 19 e 264 19 20 m 320 20 21 o 274 21 22 j 381 22 23 i 282 23 24 ▁family▁👩👩👧👦▁emoji
+zerowidth and non breaking 16 ▁ 261 0 0 z 351 0 1 er 270 1 3 o 274 3 4 ▁wi 314 4 7 d 273 7 8 th 289 8 10 ▁a 266 10 12 nd 284 12 14 ▁no 489 14 17 n 268 17 18 ▁b 309 18 20 re 349 20 22 a 279 22 23 k 354 23 24 ing 272 24 27 ▁zero▁width▁and▁non▁breaking
+quotes “fancy” and ‘single’ — dash 33 ▁quote 456 0 5 s 263 5 6 ▁ 261 6 7 <0xE2> 231 7 7 <0x80> 133 7 7 <0x9C> 161 7 8 f 337 8 9 a 279 9 10 n 268 10 11 c 411 11 12 y 275 12 13 <0xE2> 231 13 13 <0x80> 133 13 13 <0x9D> 162 13 14 ▁a 266 14 16 nd 284 16 18 ▁ 261 18 19 <0xE2> 231 19 19 <0x80> 133 19 19 <0x98> 157 19 20 s 263 20 21 ing 272 21 24 le 405 24 26 <0xE2> 231 26 26 <0x80> 133 26 26 <0x99> 158 26 27 ▁ 261 27 28 <0xE2> 231 28 28 <0x80> 133 28 28 <0x94> 153 28 29 ▁d 368 29 31 as 420 31 33 h 308 33 34 ▁quotes▁“fancy”▁and▁‘single’▁—▁dash
+ the [URL] token 8 ▁ 261 0 0 3 0 6 ▁the 265 6 10 ▁ 261 10 11 [URL] 4 11 16 ▁to 304 16 19 k 354 19 20 en 350 20 22 ▁▁the▁[URL]▁token
+a b[URL]c 6 ▁a 266 0 1 ▁ 261 1 2 3 2 8 b 344 8 9 [URL] 4 9 14 c 411 14 15 ▁a▁b[URL]c
+control tokens inline 20 ▁co 326 0 2 n 268 2 3 tro 429 3 6 l 319 6 7 ▁ 261 7 8 <0x3C> 65 8 9 s 263 9 10 <0x3E> 67 10 11 ▁to 304 11 14 k 354 14 15 en 350 15 17 s 263 17 18 ▁ 261 18 19 <0x3C> 65 19 20 <0x2F> 52 20 21 s 263 21 22 <0x3E> 67 22 23 ▁in 276 23 26 l 319 26 27 ine 415 27 30 ▁control▁▁tokens▁▁inline
+https://example.com/path?q=1&x=2 27 ▁h 512 0 1 t 269 1 2 t 269 2 3 p 352 3 4 s 263 4 5 <0x3A> 63 5 6 <0x2F> 52 6 7 <0x2F> 52 7 8 e 264 8 9 x 421 9 10 a 279 10 11 mple 485 11 15 . 262 15 16 c 411 16 17 om 475 17 19 <0x2F> 52 19 20 p 352 20 21 a 279 21 22 th 289 22 24 ? 435 24 25 q 598 25 26 <0x3D> 66 26 27 1 371 27 28 <0x26> 43 28 29 x 421 29 30 <0x3D> 66 30 31 2 434 31 32 ▁https://example.com/path?q=1&x=2
+UPPER lower MiXeD case 17 ▁ 261 0 0 U 593 0 1 P 425 1 2 P 425 2 3 E 595 3 4 R 544 4 5 ▁lo 305 5 8 w 419 8 9 er 270 9 11 ▁M 442 11 13 i 282 13 14 <0x58> 93 14 15 e 264 15 16 D 589 16 17 ▁c 338 17 19 as 420 19 21 e 264 21 22 ▁UPPER▁lower▁MiXeD▁case
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 40 ▁a 266 0 1 a 279 1 2 a 279 2 3 a 279 3 4 a 279 4 5 a 279 5 6 a 279 6 7 a 279 7 8 a 279 8 9 a 279 9 10 a 279 10 11 a 279 11 12 a 279 12 13 a 279 13 14 a 279 14 15 a 279 15 16 a 279 16 17 a 279 17 18 a 279 18 19 a 279 19 20 a 279 20 21 a 279 21 22 a 279 22 23 a 279 23 24 a 279 24 25 a 279 25 26 a 279 26 27 a 279 27 28 a 279 28 29 a 279 29 30 a 279 30 31 a 279 31 32 a 279 32 33 a 279 33 34 a 279 34 35 a 279 35 36 a 279 36 37 a 279 37 38 a 279 38 39 a 279 39 40 ▁aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+Ω≈ç√∫˜µ≤ 22 ▁ 261 0 0 <0xCE> 211 0 0 <0xA9> 174 0 1 <0xE2> 231 1 1 <0x89> 142 1 1 <0x88> 141 1 2 <0xC3> 200 2 2 <0xA7> 172 2 3 <0xE2> 231 3 3 <0x88> 141 3 3 <0x9A> 159 3 4 <0xE2> 231 4 4 <0x88> 141 4 4 <0xAB> 176 4 5 ▁ 261 5 5 <0xCC> 209 5 5 <0x83> 136 5 6 <0xCE> 211 6 6 <0xBC> 193 6 7 <0xE2> 231 7 7 <0x89> 142 7 7 <0xA4> 169 7 8 ▁Ω≈ç√∫▁̃μ≤
+مرحبا بالعالم 26 ▁ 261 0 0 <0xD9> 222 0 0 <0x85> 138 0 1 <0xD8> 221 1 1 <0xB1> 182 1 2 <0xD8> 221 2 2 <0xAD> 178 2 3 <0xD8> 221 3 3 <0xA8> 173 3 4 <0xD8> 221 4 4 <0xA7> 172 4 5 ▁ 261 5 6 <0xD8> 221 6 6 <0xA8> 173 6 7 <0xD8> 221 7 7 <0xA7> 172 7 8 <0xD9> 222 8 8 <0x84> 137 8 9 <0xD8> 221 9 9 <0xB9> 190 9 10 <0xD8> 221 10 10 <0xA7> 172 10 11 <0xD9> 222 11 11 <0x84> 137 11 12 <0xD9> 222 12 12 <0x85> 138 12 13 ▁مرحبا▁بالعالم
+ leading and trailing 10 ▁lea 499 2 5 ding 501 5 9 ▁a 266 9 11 nd 284 11 13 ▁ 261 13 14 t 269 14 15 ra 409 15 17 i 282 17 18 l 319 18 19 ing 272 19 22 ▁leading▁and▁trailing
+\ttab\tstart 5 ▁ 261 1 1 t 269 1 2 a 279 2 3 b 344 3 4 ▁start 453 4 10 ▁tab▁start
+newline\n\n\nruns 11 ▁ 261 0 0 n 268 0 1 e 264 1 2 w 419 2 3 l 319 3 4 ine 415 4 7 ▁ 261 7 10 r 288 10 11 u 271 11 12 n 268 12 13 s 263 13 14 ▁newline▁runs
+mid spaces collapse 12 ▁ 261 0 0 m 320 0 1 i 282 1 2 d 273 2 3 ▁ 261 3 6 space 389 6 11 s 263 11 12 ▁co 326 12 17 ll 412 17 19 a 279 19 20 p 352 20 21 se 325 21 23 ▁mid▁spaces▁collapse
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.model b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.model
new file mode 100644
index 0000000000..6548571f49
Binary files /dev/null and b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.model differ
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-identity.fixtures.tsv b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-identity.fixtures.tsv
new file mode 100644
index 0000000000..3677b86a84
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-identity.fixtures.tsv
@@ -0,0 +1,40 @@
+ 0
+ 0
+ 0
+a 1 ▁a 10 0 1 ▁a
+Hello world 7 ▁ 5 0 0 H 241 0 1 e 8 1 2 ll 105 2 4 o 17 4 5 ▁wor 160 5 9 ld 74 9 11 ▁Hello▁world
+ Hello world 7 ▁ 5 1 1 H 241 1 2 e 8 2 3 ll 105 3 5 o 17 5 6 ▁wor 160 6 12 ld 74 12 14 ▁Hello▁world
+Hello world.\nSecond line\ttabbed 23 ▁ 5 0 0 H 241 0 1 e 8 1 2 ll 105 2 4 o 17 4 5 ▁wor 160 5 9 ld 74 9 11 . 7 11 12 \n 0 12 13 S 297 13 14 e 8 14 15 c 38 15 16 o 17 16 17 nd 24 17 19 ▁ 5 19 20 l 30 20 21 ine 147 21 24 \t 0 24 25 t 11 25 26 a 13 26 27 b 45 27 28 b 45 28 29 ed 18 29 31 ▁Hello▁world.\nSecond▁line\ttabbed
+The quick brown fox jumps over the lazy dog. 28 ▁Th 25 0 2 e 8 2 3 ▁qu 69 3 6 i 15 6 7 ck 43 7 9 ▁b 47 9 11 r 23 11 12 ow 60 12 14 n 9 14 15 ▁fo 97 15 18 x 108 18 19 ▁ 5 19 20 j 115 20 21 u 14 21 22 m 26 22 23 p 27 23 24 s 6 24 25 ▁ 5 25 26 o 17 26 27 ve 102 27 29 r 23 29 30 ▁the 12 30 34 ▁la 88 34 37 z 54 37 38 y 19 38 39 ▁do 159 39 42 g 48 42 43 . 7 43 44 ▁The▁quick▁brown▁fox▁jumps▁over▁the▁lazy▁dog.
+tokenization and segmentation 4 ▁tokenization 200 0 12 ▁a 10 12 14 nd 24 14 16 ▁segmentation 80 16 29 ▁tokenization▁and▁segmentation
+Antidisestablishmentarianism 14 ▁An 123 0 2 t 11 2 3 i 15 3 4 d 33 4 5 is 22 5 7 est 63 7 10 a 13 10 11 b 45 11 12 lish 193 12 16 ment 209 16 20 aria 221 20 24 n 9 24 25 is 22 25 27 m 26 27 28 ▁Antidisestablishmentarianism
+water running walked faster apple book work play 15 ▁water 70 0 5 ▁runn 211 5 10 ing 20 10 13 ▁walk 87 13 18 ed 18 18 20 ▁fast 85 20 25 er 16 25 27 ▁appl 214 27 32 e 8 32 33 ▁b 47 33 35 o 17 35 36 o 17 36 37 k 66 37 38 ▁work 57 38 43 ▁play 71 43 48 ▁water▁running▁walked▁faster▁apple▁book▁work▁play
+3.14159 x 42 = 1024? 21 ▁ 5 0 0 3 225 0 1 . 7 1 2 1 109 2 3 4 110 3 4 1 109 4 5 5 238 5 6 9 239 6 7 ▁ 5 7 8 x 108 8 9 ▁ 5 9 10 4 110 10 11 2 169 11 12 ▁ 5 12 13 = 0 13 14 ▁ 5 14 15 1 109 15 16 0 282 16 17 2 169 17 18 4 110 18 19 ? 170 19 20 ▁3.14159▁x▁42▁=▁1024?
+!!!???... 10 ▁ 5 0 0 ! 113 0 1 ! 113 1 2 ! 113 2 3 ? 170 3 4 ? 170 4 5 ? 170 5 6 . 7 6 7 . 7 7 8 . 7 8 9 ▁!!!???...
+(parentheses) and [brackets] and {braces} 27 ▁ 5 0 0 ( 236 0 1 p 27 1 2 are 34 2 5 n 9 5 6 th 36 6 8 e 8 8 9 ses 150 9 12 ) 237 12 13 ▁a 10 13 15 nd 24 15 17 ▁ 5 17 18 [ 0 18 19 b 45 19 20 r 23 20 21 ack 89 21 24 e 8 24 25 ts 101 25 27 ] 0 27 28 ▁a 10 28 30 nd 24 30 32 ▁ 5 32 33 { 0 33 34 b 45 34 35 ra 152 35 37 ces 216 37 40 } 0 40 41 ▁(parentheses)▁and▁[brackets]▁and▁{braces}
+café naïve fiancé résumé 21 ▁ 5 0 0 ca 104 0 2 f 41 2 3 é 247 3 4 ▁ 5 4 5 n 9 5 6 a 13 6 7 ï 0 7 8 ve 102 8 10 ▁fi 210 10 13 a 13 13 14 n 9 14 15 c 38 15 16 é 247 16 17 ▁ 5 17 18 r 23 18 19 é 247 19 20 s 6 20 21 u 14 21 22 m 26 22 23 é 247 23 24 ▁café▁naïve▁fiancé▁résumé
+financial fluid 13 ▁ 5 0 0 fi 0 0 1 n 9 1 2 a 13 2 3 n 9 3 4 c 38 4 5 i 15 5 6 al 21 6 8 ▁ 5 8 9 fl 0 9 10 u 14 10 11 i 15 11 12 d 33 12 13 ▁financial▁fluid
+① ⑪ ㋿ ＫＡＴＡＫＡＮＡ 8 ▁ 5 0 0 ① 0 0 1 ▁ 5 1 2 ⑪ 0 2 3 ▁ 5 3 4 ㋿ 0 4 5 ▁ 5 5 6 ＫＡＴＡＫＡＮＡ 0 6 14 ▁①▁⑪▁㋿▁ＫＡＴＡＫＡＮＡ
+ｶﾀｶﾅ half width 9 ▁ 5 0 0 ｶﾀｶﾅ 0 0 4 ▁ 5 4 5 h 32 5 6 al 21 6 8 f 41 8 9 ▁wi 73 9 12 d 33 12 13 th 36 13 15 ▁ｶﾀｶﾅ▁half▁width
+東京タワーへ行きました 10 ▁ 5 0 0 東 231 0 1 京 230 1 2 タ 227 2 3 ワ 228 3 4 ー 229 4 5 へ行き 0 5 8 ま 287 8 9 し 179 9 10 た 256 10 11 ▁東京タワーへ行きました
+日本語とEnglish混在 9 ▁ 5 0 0 日 264 0 1 本 266 1 2 語 269 2 3 と 0 3 4 E 295 4 5 ng 86 5 7 lish 193 7 11 混在 0 11 13 ▁日本語とEnglish混在
+Привет мир 11 ▁ 5 0 0 П 249 0 1 р 112 1 2 и 111 2 3 в 172 3 4 е 173 4 5 т 252 5 6 ▁ 5 6 7 м 174 7 8 и 111 8 9 р 112 9 10 ▁Привет▁мир
+안녕하세요 세계 9 ▁ 5 0 0 안 234 0 1 녕 233 1 2 하 235 2 3 세 181 3 4 요 274 4 5 ▁ 5 5 6 세 181 6 7 계 271 7 8 ▁안녕하세요▁세계
+你好，世界！ 7 ▁ 5 0 0 你 261 0 1 好 262 1 2 ， 0 2 3 世 259 3 4 界 267 4 5 ！ 0 5 6 ▁你好，世界！
+I love 🍕 pizza 12 ▁ 5 0 0 I 294 0 1 ▁lo 53 1 4 ve 102 4 6 ▁ 5 6 7 🍕 279 7 9 ▁ 5 9 10 p 27 10 11 i 15 11 12 z 54 12 13 z 54 13 14 a 13 14 15 ▁I▁love▁🍕▁pizza
+flags 🇩🇪 🇺🇸 end 14 ▁ 5 0 0 f 41 0 1 l 30 1 2 a 13 2 3 g 48 3 4 s 6 4 5 ▁ 5 5 6 🇩 277 6 8 🇪 278 8 10 ▁ 5 10 11 🇺🇸 0 11 15 ▁ 5 15 16 e 8 16 17 nd 24 17 19 ▁flags▁🇩🇪▁🇺🇸▁end
+family 👩👩👧👦 emoji 10 ▁famil 185 0 5 y 19 5 6 ▁ 5 6 7 👩👩👧👦 0 7 18 ▁ 5 18 19 e 8 19 20 m 26 20 21 o 17 21 22 j 115 22 23 i 15 23 24 ▁family▁👩👩👧👦▁emoji
+zerowidth and non breaking 22 ▁ 5 0 0 z 54 0 1 er 16 1 3 o 17 3 4 0 4 5 w 78 5 6 i 15 6 7 d 33 7 8 th 36 8 10 ▁a 10 10 12 nd 24 12 14 ▁ 5 14 15 n 9 15 16 o 17 16 17 n 9 17 18 0 18 19 b 45 19 20 r 23 20 21 e 8 21 22 a 13 22 23 k 66 23 24 ing 20 24 27 ▁zerowidth▁and▁non breaking
+quotes “fancy” and ‘single’ — dash 24 ▁quote 198 0 5 s 6 5 6 ▁ 5 6 7 “ 0 7 8 f 41 8 9 a 13 9 10 n 9 10 11 c 38 11 12 y 19 12 13 ” 0 13 14 ▁a 10 14 16 nd 24 16 18 ▁ 5 18 19 ‘ 0 19 20 s 6 20 21 ing 20 21 24 le 107 24 26 ’ 0 26 27 ▁ 5 27 28 — 0 28 29 ▁d 100 29 31 a 13 31 32 s 6 32 33 h 32 33 34 ▁quotes▁“fancy”▁and▁‘single’▁—▁dash
+ the [URL] token 9 ▁ 5 0 0 3 0 6 ▁the 12 6 10 ▁ 5 10 11 [URL] 4 11 16 ▁to 51 16 19 k 66 19 20 e 8 20 21 n 9 21 22 ▁▁the▁[URL]▁token
+a b[URL]c 6 ▁a 10 0 1 ▁ 5 1 2 3 2 8 b 45 8 9 [URL] 4 9 14 c 38 14 15 ▁a▁b[URL]c
+control tokens inline 20 ▁co 99 0 2 n 9 2 3 tro 128 3 6 l 30 6 7 ▁ 5 7 8 < 0 8 9 s 6 9 10 > 0 10 11 ▁to 51 11 14 k 66 14 15 e 8 15 16 n 9 16 17 s 6 17 18 ▁ 5 18 19 0 22 23 ▁in 35 23 26 l 30 26 27 ine 147 27 30 ▁control▁▁tokens▁▁inline
+https://example.com/path?q=1&x=2 29 ▁ 5 0 0 h 32 0 1 t 11 1 2 t 11 2 3 p 27 3 4 s 6 4 5 :// 0 5 8 e 8 8 9 x 108 9 10 a 13 10 11 m 26 11 12 p 27 12 13 le 107 13 15 . 7 15 16 c 38 16 17 o 17 17 18 m 26 18 19 / 0 19 20 p 27 20 21 a 13 21 22 th 36 22 24 ? 170 24 25 q 298 25 26 = 0 26 27 1 109 27 28 & 0 28 29 x 108 29 30 = 0 30 31 2 169 31 32 ▁https://example.com/path?q=1&x=2
+UPPER lower MiXeD case 19 ▁ 5 0 0 U 293 0 1 P 156 1 2 P 156 2 3 E 295 3 4 R 242 4 5 ▁lo 53 5 8 w 78 8 9 er 16 9 11 ▁ 5 11 12 M 288 12 13 i 15 13 14 X 0 14 15 e 8 15 16 D 289 16 17 ▁ 5 17 18 ca 104 18 20 s 6 20 21 e 8 21 22 ▁UPPER▁lower▁MiXeD▁case
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 40 ▁a 10 0 1 a 13 1 2 a 13 2 3 a 13 3 4 a 13 4 5 a 13 5 6 a 13 6 7 a 13 7 8 a 13 8 9 a 13 9 10 a 13 10 11 a 13 11 12 a 13 12 13 a 13 13 14 a 13 14 15 a 13 15 16 a 13 16 17 a 13 17 18 a 13 18 19 a 13 19 20 a 13 20 21 a 13 21 22 a 13 22 23 a 13 23 24 a 13 24 25 a 13 25 26 a 13 26 27 a 13 27 28 a 13 28 29 a 13 29 30 a 13 30 31 a 13 31 32 a 13 32 33 a 13 33 34 a 13 34 35 a 13 35 36 a 13 36 37 a 13 37 38 a 13 38 39 a 13 39 40 ▁aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+Ω≈ç√∫˜µ≤ 2 ▁ 5 0 0 Ω≈ç√∫˜µ≤ 0 0 8 ▁Ω≈ç√∫˜µ≤
+مرحبا بالعالم 4 ▁ 5 0 0 مرحبا 0 0 5 ▁ 5 5 6 بالعالم 0 6 13 ▁مرحبا▁بالعالم
+ leading and trailing 13 ▁ 5 2 2 le 107 2 4 a 13 4 5 d 33 5 6 ing 20 6 9 ▁a 10 9 11 nd 24 11 13 ▁ 5 13 14 t 11 14 15 ra 152 15 17 i 15 17 18 l 30 18 19 ing 20 19 22 ▁leading▁and▁trailing
+\ttab\tstart 9 ▁ 5 0 0 \t 0 0 1 t 11 1 2 a 13 2 3 b 45 3 4 \t 0 4 5 st 46 5 7 ar 50 7 9 t 11 9 10 ▁\ttab\tstart
+newline\n\n\nruns 11 ▁ 5 0 0 n 9 0 1 e 8 1 2 w 78 2 3 l 30 3 4 ine 147 4 7 \n\n\n 0 7 10 r 23 10 11 u 14 11 12 n 9 12 13 s 6 13 14 ▁newline\n\n\nruns
+mid spaces collapse 13 ▁ 5 0 0 m 26 0 1 i 15 1 2 d 33 2 3 ▁ 5 3 6 space 127 6 11 s 6 11 12 ▁co 99 12 17 ll 105 17 19 a 13 19 20 p 27 20 21 s 6 21 22 e 8 22 23 ▁mid▁spaces▁collapse
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-identity.model b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-identity.model
new file mode 100644
index 0000000000..54c3482777
Binary files /dev/null and b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-identity.model differ
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-suffix.fixtures.tsv b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-suffix.fixtures.tsv
new file mode 100644
index 0000000000..6dbd8aad46
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-suffix.fixtures.tsv
@@ -0,0 +1,40 @@
+ 0
+ 0
+ 0
+a 1 a▁ 21 0 1 a▁
+Hello world 9 H 242 0 1 e 20 1 2 l 15 2 3 lo 59 3 5 ▁ 5 5 6 wor 71 6 9 l 15 9 10 d 16 10 11 ▁ 5 11 11 Hello▁world▁
+ Hello world 9 H 242 1 2 e 20 2 3 l 15 3 4 lo 59 4 6 ▁ 5 6 9 wor 71 9 12 l 15 12 13 d 16 13 14 ▁ 5 14 14 Hello▁world▁
+Hello world.\nSecond line\ttabbed 23 H 242 0 1 e 20 1 2 l 15 2 3 lo 59 3 5 ▁ 5 5 6 wor 71 6 9 l 15 9 10 d 16 10 11 .▁ 6 11 13 S 44 13 14 e 20 14 15 co 72 15 17 n 13 17 18 d 16 18 19 ▁ 5 19 20 l 15 20 21 in 36 21 23 e▁ 18 23 25 t 9 25 26 a 14 26 27 b 31 27 28 b 31 28 29 ed▁ 17 29 31 Hello▁world.▁Second▁line▁tabbed▁
+The quick brown fox jumps over the lazy dog. 29 The▁ 28 0 4 q 298 4 5 u 19 5 6 i 8 6 7 ck▁ 214 7 10 b 31 10 11 r 33 11 12 own▁ 65 12 16 f 35 16 17 o 10 17 18 x 149 18 19 ▁ 5 19 20 j 109 20 21 u 19 21 22 m 22 22 23 p 27 23 24 s▁ 12 24 26 o 10 26 27 v 51 27 28 er▁ 26 28 31 the▁ 11 31 35 la 98 35 37 z 78 37 38 y 37 38 39 ▁ 5 39 40 d 16 40 41 o 10 41 42 g 38 42 43 .▁ 6 43 44 The▁quick▁brown▁fox▁jumps▁over▁the▁lazy▁dog.▁
+tokenization and segmentation 8 t 9 0 1 o 10 1 2 k 41 2 3 en 39 3 5 ization▁ 150 5 13 and▁ 34 13 17 segmentation 82 17 29 ▁ 5 29 29 tokenization▁and▁segmentation▁
+Antidisestablishmentarianism 16 A 68 0 1 nti 137 1 4 d 16 4 5 i 8 5 6 s 7 6 7 est 47 7 10 a 14 10 11 b 31 11 12 lish 190 12 16 ment 229 16 20 aria 224 20 24 n 13 24 25 i 8 25 26 s 7 26 27 m 22 27 28 ▁ 5 28 28 Antidisestablishmentarianism▁
+water running walked faster apple book work play 19 water▁ 64 0 6 runn 179 6 10 ing▁ 30 10 14 walk 87 14 18 ed▁ 17 18 21 fast 88 21 25 er▁ 26 25 28 app 126 28 31 l 15 31 32 e▁ 18 32 34 b 31 34 35 o 10 35 36 o 10 36 37 k 41 37 38 ▁ 5 38 39 work 53 39 43 ▁ 5 43 44 play 66 44 48 ▁ 5 48 48 water▁running▁walked▁faster▁apple▁book▁work▁play▁
+3.14159 x 42 = 1024? 21 3 238 0 1 . 97 1 2 1 101 2 3 4 102 3 4 1 101 4 5 5 239 5 6 9 240 6 7 ▁ 5 7 8 x 149 8 9 ▁ 5 9 10 4 102 10 11 2 155 11 12 ▁ 5 12 13 = 0 13 14 ▁ 5 14 15 1 101 15 16 0 234 16 17 2 155 17 18 4 102 18 19 ? 296 19 20 ▁ 5 20 20 3.14159▁x▁42▁=▁1024?▁
+!!!???... 9 ! 297 0 1 ! 297 1 2 ! 297 2 3 ? 296 3 4 ? 296 4 5 ? 296 5 6 . 97 6 7 . 97 7 8 .▁ 6 8 9 !!!???...▁
+(parentheses) and [brackets] and {braces} 26 ( 236 0 1 par 143 1 4 en 39 4 6 the 142 6 9 se 58 9 11 s 7 11 12 ) 237 12 13 ▁ 5 13 14 and▁ 34 14 18 [ 0 18 19 b 31 19 20 ra 76 20 22 c 25 22 23 k 41 23 24 e 20 24 25 t 9 25 26 s 7 26 27 ] 0 27 28 ▁ 5 28 29 and▁ 34 29 33 { 0 33 34 b 31 34 35 ra 76 35 37 ces 218 37 40 } 0 40 41 ▁ 5 41 41 (parentheses)▁and▁[brackets]▁and▁{braces}▁
+café naïve fiancé résumé 20 ca 73 0 2 f 35 2 3 é 247 3 4 ▁ 5 4 5 na 96 5 7 ï 0 7 8 ve▁ 70 8 11 fi 74 11 13 a 14 13 14 n 13 14 15 c 25 15 16 é 247 16 17 ▁ 5 17 18 r 33 18 19 é 247 19 20 s 7 20 21 u 19 21 22 m 22 22 23 é 247 23 24 ▁ 5 24 24 café▁naïve▁fiancé▁résumé▁
+financial fluid 13 fi 74 0 1 na 96 1 3 n 13 3 4 c 25 4 5 i 8 5 6 al 23 6 8 ▁ 5 8 9 f 35 9 9 l 15 9 10 u 19 10 11 i 8 11 12 d 16 12 13 ▁ 5 13 13 financial▁fluid▁
+① ⑪ ㋿ ＫＡＴＡＫＡＮＡ 16 1 101 0 1 ▁ 5 1 2 1 101 2 2 1 101 2 3 ▁ 5 3 4 令和 0 4 5 ▁ 5 5 6 K 0 6 7 A 68 7 8 T 62 8 9 A 68 9 10 K 0 10 11 A 68 11 12 N 151 12 13 A 68 13 14 ▁ 5 14 14 1▁11▁令和▁KATAKANA▁
+ｶﾀｶﾅ half width 14 カ 0 0 1 タ 275 1 2 カナ 0 2 4 ▁ 5 4 5 h 24 5 6 al 23 6 8 f 35 8 9 ▁ 5 9 10 w 48 10 11 i 8 11 12 d 16 12 13 t 9 13 14 h 24 14 15 ▁ 5 15 15 カタカナ▁half▁width▁
+東京タワーへ行きました 10 東 284 0 1 京 280 1 2 タ 275 2 3 ワ 276 3 4 ー 277 4 5 へ行き 0 5 8 ま 294 8 9 し 177 9 10 た 254 10 11 ▁ 5 11 11 東京タワーへ行きました▁
+日本語とEnglish混在 10 日 256 0 1 本 257 1 2 語 258 2 3 と 0 3 4 E 144 4 5 n 13 5 6 g 38 6 7 lish 190 7 11 混在 0 11 13 ▁ 5 13 13 日本語とEnglish混在▁
+Привет мир 11 П 248 0 1 р 107 1 2 и 106 2 3 в 163 3 4 е 164 4 5 т 251 5 6 ▁ 5 6 7 м 165 7 8 и 106 8 9 р 107 9 10 ▁ 5 10 10 Привет▁мир▁
+안녕하세요 세계 9 안 233 0 1 녕 259 1 2 하 261 2 3 세 174 3 4 요 260 4 5 ▁ 5 5 6 세 174 6 7 계 271 7 8 ▁ 5 8 8 안녕하세요▁세계▁
+你好，世界！ 7 你 281 0 1 好 282 1 2 , 299 2 3 世 278 3 4 界 285 4 5 ! 297 5 6 ▁ 5 6 6 你好,世界!▁
+I love 🍕 pizza 11 I 145 0 1 ▁ 5 1 2 lo 59 2 4 ve▁ 70 4 7 🍕 265 7 9 ▁ 5 9 10 p 27 10 11 i 8 11 12 z 78 12 13 z 78 13 14 a▁ 21 14 15 I▁love▁🍕▁pizza▁
+flags 🇩🇪 🇺🇸 end 12 f 35 0 1 la 98 1 3 g 38 3 4 s▁ 12 4 6 🇩 263 6 8 🇪 264 8 10 ▁ 5 10 11 🇺🇸 0 11 15 ▁ 5 15 16 en 39 16 18 d 16 18 19 ▁ 5 19 19 flags▁🇩🇪▁🇺🇸▁end▁
+family 👩👩👧👦 emoji 11 famil 168 0 5 y 37 5 6 ▁ 5 6 7 👩👩👧👦 0 7 18 ▁ 5 18 19 e 20 19 20 m 22 20 21 o 10 21 22 j 109 22 23 i 8 23 24 ▁ 5 24 24 family▁👩👩👧👦▁emoji▁
+zerowidth and non breaking 18 z 78 0 1 er 50 1 3 o 10 3 4 ▁ 5 4 5 w 48 5 6 i 8 6 7 d 16 7 8 t 9 8 9 h 24 9 10 ▁ 5 10 11 and▁ 34 11 15 n 13 15 16 on▁ 132 16 19 b 31 19 20 re 40 20 22 a 14 22 23 k 41 23 24 ing▁ 30 24 27 zero▁width▁and▁non▁breaking▁
+quotes “fancy” and ‘single’ — dash 25 quote 159 0 5 s▁ 12 5 7 “ 0 7 8 f 35 8 9 a 14 9 10 n 13 10 11 c 25 11 12 y 37 12 13 ” 0 13 14 ▁ 5 14 15 and▁ 34 15 19 ‘ 0 19 20 s 7 20 21 in 36 21 23 g 38 23 24 l 15 24 25 e 20 25 26 ’ 0 26 27 ▁ 5 27 28 — 0 28 29 ▁ 5 29 30 d 16 30 31 as 77 31 33 h 24 33 34 ▁ 5 34 34 quotes▁“fancy”▁and▁‘single’▁—▁dash▁
+ the [URL] token 10 3 0 6 ▁ 5 6 7 the▁ 11 7 11 [URL] 4 11 16 ▁ 5 16 17 t 9 17 18 o 10 18 19 k 41 19 20 en 39 20 22 ▁ 5 22 22 ▁the▁[URL]▁token▁
+a b[URL]c 6 a▁ 21 0 2 3 2 8 b 31 8 9 [URL] 4 9 14 c 25 14 15 ▁ 5 15 15 a▁b[URL]c▁
+control tokens inline 21 co 72 0 2 n 13 2 3 tro 119 3 6 l▁ 61 6 8 < 0 8 9 s 7 9 10 > 0 10 11 ▁ 5 11 12 t 9 12 13 o 10 13 14 k 41 14 15 en 39 15 17 s▁ 12 17 19 0 22 23 ▁ 5 23 24 in 36 24 26 l 15 26 27 in 36 27 29 e▁ 18 29 30 control▁▁tokens▁▁inline▁
+https://example.com/path?q=1&x=2 24 h 24 0 1 t 9 1 2 t 9 2 3 p 27 3 4 s 7 4 5 :// 0 5 8 ex 120 8 10 a 14 10 11 mple 194 11 15 . 97 15 16 com 134 16 19 / 0 19 20 pa 56 20 22 t 9 22 23 h 24 23 24 ? 296 24 25 q 298 25 26 = 0 26 27 1 101 27 28 & 0 28 29 x 149 29 30 = 0 30 31 2 155 31 32 ▁ 5 32 32 https://example.com/path?q=1&x=2▁
+UPPER lower MiXeD case 18 U 230 0 1 P 80 1 2 P 80 2 3 E 144 3 4 R 235 4 5 ▁ 5 5 6 lo 59 6 8 w 48 8 9 er▁ 26 9 12 M 154 12 13 i 8 13 14 X 0 14 15 e 20 15 16 D 152 16 17 ▁ 5 17 18 ca 73 18 20 s 7 20 21 e▁ 18 21 22 UPPER▁lower▁MiXeD▁case▁
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 40 a 14 0 1 a 14 1 2 a 14 2 3 a 14 3 4 a 14 4 5 a 14 5 6 a 14 6 7 a 14 7 8 a 14 8 9 a 14 9 10 a 14 10 11 a 14 11 12 a 14 12 13 a 14 13 14 a 14 14 15 a 14 15 16 a 14 16 17 a 14 17 18 a 14 18 19 a 14 19 20 a 14 20 21 a 14 21 22 a 14 22 23 a 14 23 24 a 14 24 25 a 14 25 26 a 14 26 27 a 14 27 28 a 14 28 29 a 14 29 30 a 14 30 31 a 14 31 32 a 14 32 33 a 14 33 34 a 14 34 35 a 14 35 36 a 14 36 37 a 14 37 38 a 14 38 39 a▁ 21 39 40 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa▁
+Ω≈ç√∫˜µ≤ 4 Ω≈ç√∫ 0 0 5 ▁ 5 5 5 ̃μ≤ 0 5 8 ▁ 5 8 8 Ω≈ç√∫▁̃μ≤▁
+مرحبا بالعالم 4 مرحبا 0 0 5 ▁ 5 5 6 بالعالم 0 6 13 ▁ 5 13 13 مرحبا▁بالعالم▁
+ leading and trailing 11 l 15 2 3 e 20 3 4 a 14 4 5 d 16 5 6 ing▁ 30 6 10 and▁ 34 10 14 t 9 14 15 ra 76 15 17 i 8 17 18 l 15 18 19 ing▁ 30 19 22 leading▁and▁trailing▁
+\ttab\tstart 6 t 9 1 2 a 14 2 3 b 31 3 4 ▁ 5 4 5 start 187 5 10 ▁ 5 10 10 tab▁start▁
+newline\n\n\nruns 10 n 13 0 1 e 20 1 2 w 48 2 3 l 15 3 4 in 36 4 6 e▁ 18 6 10 r 33 10 11 u 19 11 12 n 13 12 13 s▁ 12 13 14 newline▁runs▁
+mid spaces collapse 12 m 22 0 1 i 8 1 2 d 16 2 3 ▁ 5 3 6 space 117 6 11 s▁ 12 11 15 co 72 15 17 l 15 17 18 la 98 18 20 p 27 20 21 s 7 21 22 e▁ 18 22 23 mid▁spaces▁collapse▁
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-suffix.model b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-suffix.model
new file mode 100644
index 0000000000..984acd070e
Binary files /dev/null and b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-suffix.model differ
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.fixtures.tsv b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.fixtures.tsv
new file mode 100644
index 0000000000..5eeffe47e5
--- /dev/null
+++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.fixtures.tsv
@@ -0,0 +1,40 @@
+ 0
+ 0
+ 0
+a 1 ▁a 10 0 1 ▁a
+Hello world 7 ▁ 5 0 0 H 241 0 1 e 8 1 2 ll 105 2 4 o 17 4 5 ▁wor 160 5 9 ld 74 9 11 ▁Hello▁world
+ Hello world 7 ▁ 5 1 1 H 241 1 2 e 8 2 3 ll 105 3 5 o 17 5 6 ▁wor 160 6 12 ld 74 12 14 ▁Hello▁world
+Hello world.\nSecond line\ttabbed 23 ▁ 5 0 0 H 241 0 1 e 8 1 2 ll 105 2 4 o 17 4 5 ▁wor 160 5 9 ld 74 9 11 . 7 11 12 ▁ 5 12 13 S 297 13 14 e 8 14 15 c 38 15 16 o 17 16 17 nd 24 17 19 ▁ 5 19 20 l 30 20 21 ine 147 21 24 ▁ 5 24 25 t 11 25 26 a 13 26 27 b 45 27 28 b 45 28 29 ed 18 29 31 ▁Hello▁world.▁Second▁line▁tabbed
+The quick brown fox jumps over the lazy dog. 28 ▁Th 25 0 2 e 8 2 3 ▁qu 69 3 6 i 15 6 7 ck 43 7 9 ▁b 47 9 11 r 23 11 12 ow 60 12 14 n 9 14 15 ▁fo 97 15 18 x 108 18 19 ▁ 5 19 20 j 115 20 21 u 14 21 22 m 26 22 23 p 27 23 24 s 6 24 25 ▁ 5 25 26 o 17 26 27 ve 102 27 29 r 23 29 30 ▁the 12 30 34 ▁la 88 34 37 z 54 37 38 y 19 38 39 ▁do 159 39 42 g 48 42 43 . 7 43 44 ▁The▁quick▁brown▁fox▁jumps▁over▁the▁lazy▁dog.
+tokenization and segmentation 4 ▁tokenization 200 0 12 ▁a 10 12 14 nd 24 14 16 ▁segmentation 80 16 29 ▁tokenization▁and▁segmentation
+Antidisestablishmentarianism 14 ▁An 123 0 2 t 11 2 3 i 15 3 4 d 33 4 5 is 22 5 7 est 63 7 10 a 13 10 11 b 45 11 12 lish 193 12 16 ment 209 16 20 aria 221 20 24 n 9 24 25 is 22 25 27 m 26 27 28 ▁Antidisestablishmentarianism
+water running walked faster apple book work play 15 ▁water 70 0 5 ▁runn 211 5 10 ing 20 10 13 ▁walk 87 13 18 ed 18 18 20 ▁fast 85 20 25 er 16 25 27 ▁appl 214 27 32 e 8 32 33 ▁b 47 33 35 o 17 35 36 o 17 36 37 k 66 37 38 ▁work 57 38 43 ▁play 71 43 48 ▁water▁running▁walked▁faster▁apple▁book▁work▁play
+3.14159 x 42 = 1024? 21 ▁ 5 0 0 3 225 0 1 . 7 1 2 1 109 2 3 4 110 3 4 1 109 4 5 5 238 5 6 9 239 6 7 ▁ 5 7 8 x 108 8 9 ▁ 5 9 10 4 110 10 11 2 169 11 12 ▁ 5 12 13 = 0 13 14 ▁ 5 14 15 1 109 15 16 0 282 16 17 2 169 17 18 4 110 18 19 ? 170 19 20 ▁3.14159▁x▁42▁=▁1024?
+!!!???... 10 ▁ 5 0 0 ! 113 0 1 ! 113 1 2 ! 113 2 3 ? 170 3 4 ? 170 4 5 ? 170 5 6 . 7 6 7 . 7 7 8 . 7 8 9 ▁!!!???...
+(parentheses) and [brackets] and {braces} 27 ▁ 5 0 0 ( 236 0 1 p 27 1 2 are 34 2 5 n 9 5 6 th 36 6 8 e 8 8 9 ses 150 9 12 ) 237 12 13 ▁a 10 13 15 nd 24 15 17 ▁ 5 17 18 [ 0 18 19 b 45 19 20 r 23 20 21 ack 89 21 24 e 8 24 25 ts 101 25 27 ] 0 27 28 ▁a 10 28 30 nd 24 30 32 ▁ 5 32 33 { 0 33 34 b 45 34 35 ra 152 35 37 ces 216 37 40 } 0 40 41 ▁(parentheses)▁and▁[brackets]▁and▁{braces}
+café naïve fiancé résumé 21 ▁ 5 0 0 ca 104 0 2 f 41 2 3 é 247 3 4 ▁ 5 4 5 n 9 5 6 a 13 6 7 ï 0 7 8 ve 102 8 10 ▁fi 210 10 13 a 13 13 14 n 9 14 15 c 38 15 16 é 247 16 17 ▁ 5 17 18 r 23 18 19 é 247 19 20 s 6 20 21 u 14 21 22 m 26 22 23 é 247 23 24 ▁café▁naïve▁fiancé▁résumé
+financial fluid 13 ▁fi 210 0 1 n 9 1 2 a 13 2 3 n 9 3 4 c 38 4 5 i 15 5 6 al 21 6 8 ▁ 5 8 9 f 41 9 9 l 30 9 10 u 14 10 11 i 15 11 12 d 33 12 13 ▁financial▁fluid
+① ⑪ ㋿ ＫＡＴＡＫＡＮＡ 16 ▁ 5 0 0 1 109 0 1 ▁ 5 1 2 1 109 2 2 1 109 2 3 ▁ 5 3 4 令和 0 4 5 ▁ 5 5 6 K 0 6 7 A 296 7 8 T 299 8 9 A 296 9 10 K 0 10 11 A 296 11 12 N 224 12 13 A 296 13 14 ▁1▁11▁令和▁KATAKANA
+ｶﾀｶﾅ half width 11 ▁ 5 0 0 カ 0 0 1 タ 227 1 2 カナ 0 2 4 ▁ 5 4 5 h 32 5 6 al 21 6 8 f 41 8 9 ▁wi 73 9 12 d 33 12 13 th 36 13 15 ▁カタカナ▁half▁width
+東京タワーへ行きました 10 ▁ 5 0 0 東 231 0 1 京 230 1 2 タ 227 2 3 ワ 228 3 4 ー 229 4 5 へ行き 0 5 8 ま 287 8 9 し 179 9 10 た 256 10 11 ▁東京タワーへ行きました
+日本語とEnglish混在 9 ▁ 5 0 0 日 264 0 1 本 266 1 2 語 269 2 3 と 0 3 4 E 295 4 5 ng 86 5 7 lish 193 7 11 混在 0 11 13 ▁日本語とEnglish混在
+Привет мир 11 ▁ 5 0 0 П 249 0 1 р 112 1 2 и 111 2 3 в 172 3 4 е 173 4 5 т 252 5 6 ▁ 5 6 7 м 174 7 8 и 111 8 9 р 112 9 10 ▁Привет▁мир
+안녕하세요 세계 9 ▁ 5 0 0 안 234 0 1 녕 233 1 2 하 235 2 3 세 181 3 4 요 274 4 5 ▁ 5 5 6 세 181 6 7 계 271 7 8 ▁안녕하세요▁세계
+你好，世界！ 7 ▁ 5 0 0 你 261 0 1 好 262 1 2 , 31 2 3 世 259 3 4 界 267 4 5 ! 113 5 6 ▁你好,世界!
+I love 🍕 pizza 12 ▁ 5 0 0 I 294 0 1 ▁lo 53 1 4 ve 102 4 6 ▁ 5 6 7 🍕 279 7 9 ▁ 5 9 10 p 27 10 11 i 15 11 12 z 54 12 13 z 54 13 14 a 13 14 15 ▁I▁love▁🍕▁pizza
+flags 🇩🇪 🇺🇸 end 14 ▁ 5 0 0 f 41 0 1 l 30 1 2 a 13 2 3 g 48 3 4 s 6 4 5 ▁ 5 5 6 🇩 277 6 8 🇪 278 8 10 ▁ 5 10 11 🇺🇸 0 11 15 ▁ 5 15 16 e 8 16 17 nd 24 17 19 ▁flags▁🇩🇪▁🇺🇸▁end
+family 👩👩👧👦 emoji 10 ▁famil 185 0 5 y 19 5 6 ▁ 5 6 7 👩👩👧👦 0 7 18 ▁ 5 18 19 e 8 19 20 m 26 20 21 o 17 21 22 j 115 22 23 i 15 23 24 ▁family▁👩👩👧👦▁emoji
+zerowidth and non breaking 19 ▁ 5 0 0 z 54 0 1 er 16 1 3 o 17 3 4 ▁wi 73 4 7 d 33 7 8 th 36 8 10 ▁a 10 10 12 nd 24 12 14 ▁ 5 14 15 n 9 15 16 o 17 16 17 n 9 17 18 ▁b 47 18 20 r 23 20 21 e 8 21 22 a 13 22 23 k 66 23 24 ing 20 24 27 ▁zero▁width▁and▁non▁breaking
+quotes “fancy” and ‘single’ — dash 24 ▁quote 198 0 5 s 6 5 6 ▁ 5 6 7 “ 0 7 8 f 41 8 9 a 13 9 10 n 9 10 11 c 38 11 12 y 19 12 13 ” 0 13 14 ▁a 10 14 16 nd 24 16 18 ▁ 5 18 19 ‘ 0 19 20 s 6 20 21 ing 20 21 24 le 107 24 26 ’ 0 26 27 ▁ 5 27 28 — 0 28 29 ▁d 100 29 31 a 13 31 32 s 6 32 33 h 32 33 34 ▁quotes▁“fancy”▁and▁‘single’▁—▁dash
+ the [URL] token 9 ▁ 5 0 0 3 0 6 ▁the 12 6 10 ▁ 5 10 11 [URL] 4 11 16 ▁to 51 16 19 k 66 19 20 e 8 20 21 n 9 21 22 ▁▁the▁[URL]▁token
+a b[URL]c 6 ▁a 10 0 1 ▁ 5 1 2 3 2 8 b 45 8 9 [URL] 4 9 14 c 38 14 15 ▁a▁b[URL]c
+control tokens inline 20 ▁co 99 0 2 n 9 2 3 tro 128 3 6 l 30 6 7 ▁ 5 7 8 < 0 8 9 s 6 9 10 > 0 10 11 ▁to 51 11 14 k 66 14 15 e 8 15 16 n 9 16 17 s 6 17 18 ▁ 5 18 19 0 22 23 ▁in 35 23 26 l 30 26 27 ine 147 27 30 ▁control▁▁tokens▁▁inline
+https://example.com/path?q=1&x=2 29 ▁ 5 0 0 h 32 0 1 t 11 1 2 t 11 2 3 p 27 3 4 s 6 4 5 :// 0 5 8 e 8 8 9 x 108 9 10 a 13 10 11 m 26 11 12 p 27 12 13 le 107 13 15 . 7 15 16 c 38 16 17 o 17 17 18 m 26 18 19 / 0 19 20 p 27 20 21 a 13 21 22 th 36 22 24 ? 170 24 25 q 298 25 26 = 0 26 27 1 109 27 28 & 0 28 29 x 108 29 30 = 0 30 31 2 169 31 32 ▁https://example.com/path?q=1&x=2
+UPPER lower MiXeD case 19 ▁ 5 0 0 U 293 0 1 P 156 1 2 P 156 2 3 E 295 3 4 R 242 4 5 ▁lo 53 5 8 w 78 8 9 er 16 9 11 ▁ 5 11 12 M 288 12 13 i 15 13 14 X 0 14 15 e 8 15 16 D 289 16 17 ▁ 5 17 18 ca 104 18 20 s 6 20 21 e 8 21 22 ▁UPPER▁lower▁MiXeD▁case
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 40 ▁a 10 0 1 a 13 1 2 a 13 2 3 a 13 3 4 a 13 4 5 a 13 5 6 a 13 6 7 a 13 7 8 a 13 8 9 a 13 9 10 a 13 10 11 a 13 11 12 a 13 12 13 a 13 13 14 a 13 14 15 a 13 15 16 a 13 16 17 a 13 17 18 a 13 18 19 a 13 19 20 a 13 20 21 a 13 21 22 a 13 22 23 a 13 23 24 a 13 24 25 a 13 25 26 a 13 26 27 a 13 27 28 a 13 28 29 a 13 29 30 a 13 30 31 a 13 31 32 a 13 32 33 a 13 33 34 a 13 34 35 a 13 35 36 a 13 36 37 a 13 37 38 a 13 38 39 a 13 39 40 ▁aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+Ω≈ç√∫˜µ≤ 4 ▁ 5 0 0 Ω≈ç√∫ 0 0 5 ▁ 5 5 5 ̃μ≤ 0 5 8 ▁Ω≈ç√∫▁̃μ≤
+مرحبا بالعالم 4 ▁ 5 0 0 مرحبا 0 0 5 ▁ 5 5 6 بالعالم 0 6 13 ▁مرحبا▁بالعالم
+ leading and trailing 13 ▁ 5 2 2 le 107 2 4 a 13 4 5 d 33 5 6 ing 20 6 9 ▁a 10 9 11 nd 24 11 13 ▁ 5 13 14 t 11 14 15 ra 152 15 17 i 15 17 18 l 30 18 19 ing 20 19 22 ▁leading▁and▁trailing
+\ttab\tstart 5 ▁ 5 1 1 t 11 1 2 a 13 2 3 b 45 3 4 ▁start 191 4 10 ▁tab▁start
+newline\n\n\nruns 11 ▁ 5 0 0 n 9 0 1 e 8 1 2 w 78 2 3 l 30 3 4 ine 147 4 7 ▁ 5 7 10 r 23 10 11 u 14 11 12 n 9 12 13 s 6 13 14 ▁newline▁runs
+mid spaces collapse 13 ▁ 5 0 0 m 26 0 1 i 15 1 2 d 33 2 3 ▁ 5 3 6 space 127 6 11 s 6 11 12 ▁co 99 12 17 ll 105 17 19 a 13 19 20 p 27 20 21 s 6 21 22 e 8 22 23 ▁mid▁spaces▁collapse
diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.model b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.model
new file mode 100644
index 0000000000..b6e30611e4
Binary files /dev/null and b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.model differ
diff --git a/opennlp-extensions/pom.xml b/opennlp-extensions/pom.xml
index 9afcd3fe3c..dd233553e0 100644
--- a/opennlp-extensions/pom.xml
+++ b/opennlp-extensions/pom.xml
@@ -39,6 +39,7 @@
opennlp-morfologik
+ opennlp-subword
opennlp-spellcheck
opennlp-uima
diff --git a/rat-excludes b/rat-excludes
index 5a5d86b90c..a505ab2d60 100644
--- a/rat-excludes
+++ b/rat-excludes
@@ -70,3 +70,8 @@ src/main/resources/opennlp/tools/tokenize/uax29/WordBreakProperty.txt
src/main/resources/opennlp/tools/tokenize/uax29/ExtendedPictographic.txt
src/main/resources/opennlp/tools/util/normalizer/confusables.txt
src/test/resources/opennlp/tools/tokenize/uax29/WordBreakTest.txt
+
+
+src/test/resources/opennlp/subword/sentencepiece/*.model
+src/test/resources/opennlp/subword/sentencepiece/*.fixtures.tsv
+src/test/resources/opennlp/subword/sentencepiece/corpus.txt