From b892c9d4892d7e9e1a26328e4650ac938db2e3b1 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 13:30:41 -0400 Subject: [PATCH 01/14] OPENNLP-1894: Lattice segmentation over user-supplied mecab-format dictionaries A Viterbi decoder over word and connection costs segments languages written without spaces; the same engine serves Japanese and Korean because the language lives entirely in the dictionary. Unknown text is handled through the dictionary's character categories, and every span stays in original text coordinates. An installer fetches and unpacks a user-chosen dictionary archive at install time: nothing is bundled, no location is built in, and entry names are flattened so no archive path escapes the target directory. (cherry picked from commit a699c8aeaffd107a21e4ba8ed281582366d92bb8) --- .../tokenize/lattice/LatticeTokenizer.java | 266 ++++++++++++++ .../tokenize/lattice/MecabDictionary.java | 346 ++++++++++++++++++ .../lattice/MecabDictionaryInstaller.java | 226 ++++++++++++ .../tools/tokenize/lattice/Morpheme.java | 64 ++++ .../lattice/LatticeTokenizerTest.java | 153 ++++++++ .../lattice/MecabDictionaryInstallerTest.java | 133 +++++++ 6 files changed, 1188 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java new file mode 100644 index 0000000000..c022374a5e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.util.ArrayList; +import java.util.List; + +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.tokenize.lattice.MecabDictionary.Category; +import opennlp.tools.tokenize.lattice.MecabDictionary.WordEntry; +import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; + +/** + * Dictionary-driven segmentation for languages written without spaces: a Viterbi + * search over the word lattice of a {@link MecabDictionary}, minimizing the sum of + * word costs and connection costs. This is the segmentation approach behind Japanese + * and Korean morphological analysis; the same decoder serves both, since the language + * lives entirely in the user-supplied dictionary. + * + *

Unknown text is handled through the dictionary's character categories: where the + * lexicon has no entry, or a category always invokes them, unknown-word candidates are + * generated per category template, grouping runs of same-category characters when the + * category says so. Whitespace never joins a morpheme and is never reported as one. + * Every reported span is in original text coordinates.

+ * + *

{@link #analyze(String)} returns full morphemes with their dictionary features; + * the {@link Tokenizer} view reports just the surfaces and spans.

+ * + *

The tokenizer reads only immutable dictionary state and is safe to share between + * threads.

+ * + * @since 3.0.0 + */ +public class LatticeTokenizer implements Tokenizer { + + /** The context id of the beginning and end of text. */ + private static final int BOUNDARY_CONTEXT = 0; + + private final MecabDictionary dictionary; + + /** + * Initializes the tokenizer. + * + * @param dictionary The dictionary to segment with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code dictionary} is {@code null}. + */ + public LatticeTokenizer(MecabDictionary dictionary) { + if (dictionary == null) { + throw new IllegalArgumentException("dictionary must not be null"); + } + this.dictionary = dictionary; + } + + /** One lattice node: a candidate morpheme with its best path cost so far. */ + private static final class Node { + private final int start; + private final int end; + private final WordEntry entry; + private final boolean unknown; + private long pathCost = Long.MAX_VALUE; + private Node previous; + + private Node(int start, int end, WordEntry entry, boolean unknown) { + this.start = start; + this.end = end; + this.entry = entry; + this.unknown = unknown; + } + } + + /** + * Segments a text into morphemes with their dictionary features. + * + * @param text The text to segment. Must not be {@code null}. + * @return The morphemes in text order, spans in original coordinates, whitespace + * omitted. Never {@code null}; empty for empty or all-whitespace input. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + public List analyze(String text) { + if (text == null) { + throw new IllegalArgumentException("text must not be null"); + } + final List morphemes = new ArrayList<>(); + int start = 0; + while (start < text.length()) { + if (StringUtil.isWhitespace(text.charAt(start))) { + start++; + continue; + } + int end = start; + while (end < text.length() && !StringUtil.isWhitespace(text.charAt(end))) { + end++; + } + decode(text, start, end, morphemes); + start = end; + } + return morphemes; + } + + @Override + public String[] tokenize(String s) { + final List morphemes = analyze(s); + final String[] tokens = new String[morphemes.size()]; + for (int i = 0; i < tokens.length; i++) { + tokens[i] = morphemes.get(i).surface(); + } + return tokens; + } + + @Override + public Span[] tokenizePos(String s) { + final List morphemes = analyze(s); + final Span[] spans = new Span[morphemes.size()]; + for (int i = 0; i < spans.length; i++) { + spans[i] = morphemes.get(i).span(); + } + return spans; + } + + /** Runs the Viterbi search over one whitespace-free stretch of text. */ + private void decode(String text, int from, int to, List morphemes) { + final int length = to - from; + final List> endingAt = new ArrayList<>(length + 1); + for (int i = 0; i <= length; i++) { + endingAt.add(new ArrayList<>()); + } + final boolean[] reachable = new boolean[length + 1]; + reachable[0] = true; + + for (int i = 0; i < length; i++) { + if (!reachable[i]) { + continue; + } + final List candidates = candidates(text, from, to, i); + for (final Node candidate : candidates) { + relax(candidate, i == 0 ? null : endingAt.get(i)); + if (candidate.pathCost < Long.MAX_VALUE) { + endingAt.get(candidate.end - from).add(candidate); + reachable[candidate.end - from] = true; + } + } + } + + Node best = null; + for (final Node node : endingAt.get(length)) { + final long total = node.pathCost + + dictionary.connectionCost(node.entry.rightId(), BOUNDARY_CONTEXT); + if (best == null || total < best.pathCost + + dictionary.connectionCost(best.entry.rightId(), BOUNDARY_CONTEXT)) { + best = node; + } + } + if (best == null) { + throw new IllegalStateException( + "no segmentation path for \"" + text.subSequence(from, to) + "\""); + } + + final List reversed = new ArrayList<>(); + for (Node node = best; node != null; node = node.previous) { + reversed.add(new Morpheme(new Span(node.start, node.end), + text.substring(node.start, node.end), node.entry.features(), node.unknown)); + } + for (int i = reversed.size() - 1; i >= 0; i--) { + morphemes.add(reversed.get(i)); + } + } + + /** Connects a candidate to the cheapest predecessor ending where it starts. */ + private void relax(Node candidate, List predecessors) { + if (predecessors == null) { + candidate.pathCost = candidate.entry.cost() + + dictionary.connectionCost(BOUNDARY_CONTEXT, candidate.entry.leftId()); + return; + } + for (final Node predecessor : predecessors) { + final long total = predecessor.pathCost + + dictionary.connectionCost(predecessor.entry.rightId(), candidate.entry.leftId()) + + candidate.entry.cost(); + if (total < candidate.pathCost) { + candidate.pathCost = total; + candidate.previous = predecessor; + } + } + } + + /** Gathers lexicon matches and unknown-word candidates starting at one position. */ + private List candidates(String text, int from, int to, int offset) { + final int position = from + offset; + final List candidates = new ArrayList<>(); + final int longest = Math.min(dictionary.maxSurfaceLength(), to - position); + boolean lexiconMatch = false; + for (int length = 1; length <= longest; length++) { + final List entries = + dictionary.lookup(text.substring(position, position + length)); + if (entries == null) { + continue; + } + lexiconMatch = true; + for (final WordEntry entry : entries) { + candidates.add(new Node(position, position + length, entry, false)); + } + } + + final Category category = dictionary.categoryOf(text.charAt(position)); + if (!lexiconMatch || category.invoke()) { + int run = position + 1; + while (run < to + && dictionary.categoryOf(text.charAt(run)).name().equals(category.name())) { + run++; + } + final List templates = dictionary.unknownEntries(category.name()); + if (templates != null) { + addUnknown(candidates, position, run, to, category, templates); + } + } + if (candidates.isEmpty()) { + // no entry and no template: a single-character fallback keeps the lattice alive + final List fallback = dictionary.unknownEntries("DEFAULT"); + if (fallback != null) { + for (final WordEntry entry : fallback) { + candidates.add(new Node(position, position + 1, entry, true)); + } + } + } + if (candidates.isEmpty()) { + throw new IllegalStateException("dictionary provides no candidate at position " + + position + "; unk.def lacks a DEFAULT template"); + } + return candidates; + } + + /** Emits unknown-word candidates per the category's grouping and length settings. */ + private static void addUnknown(List candidates, int position, int runEnd, + int to, Category category, List templates) { + if (category.group()) { + for (final WordEntry entry : templates) { + candidates.add(new Node(position, runEnd, entry, true)); + } + } + final int lengths = category.length(); + for (int length = 1; length <= lengths && position + length <= to; length++) { + if (category.group() && position + length == runEnd) { + continue; // already emitted as the grouped run + } + for (final WordEntry entry : templates) { + candidates.add(new Node(position, position + length, entry, true)); + } + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java new file mode 100644 index 0000000000..5e1c60f1a6 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.util.StringUtil; + +/** + * An immutable, in-memory dictionary in the mecab directory format: lexicon entries + * from the {@code *.csv} files, connection costs from {@code matrix.def}, character + * categories from {@code char.def}, and unknown-word templates from {@code unk.def}, + * loaded from a user-supplied dictionary directory. The reader is a clean-room + * implementation of the documented format; no dictionary data is bundled or downloaded + * by this class, so the dictionaries' own licenses never attach to this library. + * + *

The same format serves multiple languages: the Japanese IPADIC and UniDic + * distributions and the Korean mecab-ko-dic all load through this one reader, with the + * feature columns passed through untouched because their schemas differ.

+ * + *

Instances are immutable and safe to share between threads.

+ * + * @see LatticeTokenizer + * @since 3.0.0 + */ +public final class MecabDictionary { + + /** One lexicon or unknown-word entry. */ + record WordEntry(int leftId, int rightId, int cost, List features) { + } + + /** One character category's unknown-word behavior from {@code char.def}. */ + record Category(String name, boolean invoke, boolean group, int length) { + } + + private final Map> lexicon; + private final int maxSurfaceLength; + private final short[] connectionCosts; + private final int rightSize; + private final Map categories; + private final String[] categoryOfChar; + private final Map> unknownEntries; + + private MecabDictionary(Map> lexicon, int maxSurfaceLength, + short[] connectionCosts, int rightSize, Map categories, + String[] categoryOfChar, Map> unknownEntries) { + this.lexicon = lexicon; + this.maxSurfaceLength = maxSurfaceLength; + this.connectionCosts = connectionCosts; + this.rightSize = rightSize; + this.categories = categories; + this.categoryOfChar = categoryOfChar; + this.unknownEntries = unknownEntries; + } + + /** + * Loads a dictionary directory encoded in UTF-8. + * + * @param directory The unpacked dictionary directory. Must not be {@code null}. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if reading fails or a file is malformed. + * @throws IllegalArgumentException Thrown if {@code directory} is {@code null}. + */ + public static MecabDictionary load(Path directory) throws IOException { + return load(directory, StandardCharsets.UTF_8); + } + + /** + * Loads a dictionary directory. + * + * @param directory The unpacked dictionary directory holding the {@code *.csv} + * lexicon files, {@code matrix.def}, {@code char.def}, and + * {@code unk.def}. Must not be {@code null}. + * @param charset The encoding the distribution uses, for example UTF-8 or EUC-JP. + * Must not be {@code null}. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if reading fails, a required file is missing, or a file + * is malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static MecabDictionary load(Path directory, Charset charset) throws IOException { + if (directory == null || charset == null) { + throw new IllegalArgumentException("directory and charset must not be null"); + } + final Map> lexicon = new HashMap<>(); + int maxSurface = 0; + try (DirectoryStream csvFiles = Files.newDirectoryStream(directory, "*.csv")) { + for (final Path csv : csvFiles) { + maxSurface = Math.max(maxSurface, readLexicon(csv, charset, lexicon)); + } + } + if (lexicon.isEmpty()) { + throw new IOException("no lexicon entries found under " + directory); + } + + final List matrixLines = readLines(directory.resolve("matrix.def"), charset); + if (matrixLines.isEmpty()) { + throw new IOException("empty matrix.def under " + directory); + } + final String[] header = splitWhitespace(matrixLines.get(0)); + if (header.length != 2) { + throw new IOException("malformed matrix.def header: " + matrixLines.get(0)); + } + final int leftSize = parseInt(header[0], "matrix.def", 1); + final int rightSize = parseInt(header[1], "matrix.def", 1); + final short[] costs = new short[leftSize * rightSize]; + for (int i = 1; i < matrixLines.size(); i++) { + final String line = matrixLines.get(i).trim(); + if (line.isEmpty()) { + continue; + } + final String[] fields = splitWhitespace(line); + if (fields.length != 3) { + throw new IOException("malformed matrix.def line " + (i + 1)); + } + final int right = parseInt(fields[0], "matrix.def", i + 1); + final int left = parseInt(fields[1], "matrix.def", i + 1); + costs[right * rightSize + left] = (short) parseInt(fields[2], "matrix.def", i + 1); + } + + final Map categories = new HashMap<>(); + final String[] categoryOfChar = new String[Character.MAX_VALUE + 1]; + readCharacterDefinition(directory.resolve("char.def"), charset, categories, + categoryOfChar); + + final Map> unknown = new HashMap<>(); + readLexicon(directory.resolve("unk.def"), charset, unknown); + + return new MecabDictionary(lexicon, maxSurface, costs, rightSize, categories, + categoryOfChar, unknown); + } + + /** Reads one lexicon-format CSV file; returns the longest surface seen. */ + private static int readLexicon(Path file, Charset charset, + Map> target) throws IOException { + int maxSurface = 0; + int lineNumber = 0; + for (final String line : readLines(file, charset)) { + lineNumber++; + if (line.isEmpty()) { + continue; + } + final List fields = splitCsv(line); + if (fields.size() < 4) { + throw new IOException("malformed entry at " + file + " line " + lineNumber); + } + final String surface = fields.get(0); + if (surface.isEmpty()) { + continue; + } + final WordEntry entry = new WordEntry( + parseInt(fields.get(1), file.toString(), lineNumber), + parseInt(fields.get(2), file.toString(), lineNumber), + parseInt(fields.get(3), file.toString(), lineNumber), + List.copyOf(fields.subList(4, fields.size()))); + target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); + maxSurface = Math.max(maxSurface, surface.length()); + } + return maxSurface; + } + + /** Reads char.def: category behavior lines and code point mapping lines. */ + private static void readCharacterDefinition(Path file, Charset charset, + Map categories, String[] categoryOfChar) throws IOException { + int lineNumber = 0; + for (final String raw : readLines(file, charset)) { + lineNumber++; + final String line = stripComment(raw).trim(); + if (line.isEmpty()) { + continue; + } + final String[] fields = splitWhitespace(line); + if (fields[0].startsWith("0x") || fields[0].startsWith("0X")) { + final int rangeSeparator = fields[0].indexOf(".."); + final int from; + final int to; + if (rangeSeparator >= 0) { + from = parseCodePoint(fields[0].substring(0, rangeSeparator), file, lineNumber); + to = parseCodePoint(fields[0].substring(rangeSeparator + 2), file, lineNumber); + } else { + from = parseCodePoint(fields[0], file, lineNumber); + to = from; + } + if (fields.length < 2) { + throw new IOException("mapping without category at " + file + " line " + lineNumber); + } + for (int c = from; c <= to && c <= Character.MAX_VALUE; c++) { + categoryOfChar[c] = fields[1]; + } + } else { + if (fields.length < 4) { + throw new IOException("malformed category at " + file + " line " + lineNumber); + } + categories.put(fields[0], new Category(fields[0], + "1".equals(fields[1]), "1".equals(fields[2]), + parseInt(fields[3], file.toString(), lineNumber))); + } + } + if (!categories.containsKey("DEFAULT")) { + throw new IOException("char.def defines no DEFAULT category: " + file); + } + } + + /** + * Looks up the lexicon entries for an exact surface form. + * + * @param surface The surface form. + * @return The entries, or {@code null} when the surface is not listed. + */ + List lookup(String surface) { + return lexicon.get(surface); + } + + /** @return The longest surface form in the lexicon, bounding prefix enumeration. */ + int maxSurfaceLength() { + return maxSurfaceLength; + } + + /** + * Reads the connection cost between two adjacent nodes. + * + * @param rightId The right context id of the earlier node. + * @param leftId The left context id of the later node. + * @return The connection cost. + */ + int connectionCost(int rightId, int leftId) { + return connectionCosts[rightId * rightSize + leftId]; + } + + /** + * Classifies a character. + * + * @param c The character. + * @return Its category, falling back to {@code DEFAULT}. Never {@code null}. + */ + Category categoryOf(char c) { + final String name = categoryOfChar[c]; + final Category category = name == null ? null : categories.get(name); + return category != null ? category : categories.get("DEFAULT"); + } + + /** + * Looks up the unknown-word templates of a category. + * + * @param category The category name. + * @return The templates, or {@code null} when the category has none. + */ + List unknownEntries(String category) { + return unknownEntries.get(category); + } + + private static List readLines(Path file, Charset charset) throws IOException { + if (!Files.exists(file)) { + throw new IOException("required dictionary file is missing: " + file); + } + final String content = new String(Files.readAllBytes(file), charset); + final List lines = new ArrayList<>(); + int start = 0; + for (int i = 0; i <= content.length(); i++) { + if (i == content.length() || content.charAt(i) == '\n') { + int end = i; + if (end > start && content.charAt(end - 1) == '\r') { + end--; + } + lines.add(content.substring(start, end)); + start = i + 1; + } + } + return lines; + } + + private static String stripComment(String line) { + final int hash = line.indexOf('#'); + return hash < 0 ? line : line.substring(0, hash); + } + + private static List splitCsv(String line) { + final List fields = new ArrayList<>(); + int start = 0; + for (int i = 0; i <= line.length(); i++) { + if (i == line.length() || line.charAt(i) == ',') { + fields.add(line.substring(start, i)); + start = i + 1; + } + } + return fields; + } + + private static String[] splitWhitespace(String line) { + final List parts = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= line.length(); i++) { + if (i == line.length() || StringUtil.isWhitespace(line.charAt(i))) { + if (start >= 0) { + parts.add(line.substring(start, i)); + start = -1; + } + } else if (start < 0) { + start = i; + } + } + return parts.toArray(new String[0]); + } + + private static int parseInt(String text, String file, int lineNumber) + throws IOException { + try { + return Integer.parseInt(text.trim()); + } catch (NumberFormatException e) { + throw new IOException("malformed number in " + file + " line " + lineNumber, e); + } + } + + private static int parseCodePoint(String text, Path file, int lineNumber) + throws IOException { + try { + return Integer.parseInt(text.trim().substring(2), 16); + } catch (RuntimeException e) { + throw new IOException("malformed code point in " + file + " line " + lineNumber, e); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java new file mode 100644 index 0000000000..33f3ef58dd --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.zip.GZIPInputStream; + +/** + * Fetches and unpacks a mecab-format dictionary archive into a local directory, so the + * dictionary is acquired by the user at install time and never ships with this library. + * The user supplies the archive location from the dictionary project of their choice + * and thereby accepts that dictionary's license; nothing is bundled and no location is + * built in. + * + *

The installer reads gzip-compressed tar archives, the format the common + * distributions use, and extracts only the files a {@link MecabDictionary} reads: + * {@code *.csv}, {@code *.def}, and {@code dicrc}. Entries are flattened to their base + * names, which also means no archive path can escape the target directory.

+ * + * @since 3.0.0 + */ +public final class MecabDictionaryInstaller { + + private static final int TAR_BLOCK = 512; + private static final int TAR_NAME_LENGTH = 100; + private static final int TAR_SIZE_OFFSET = 124; + private static final int TAR_SIZE_LENGTH = 12; + private static final int TAR_TYPE_OFFSET = 156; + + private MecabDictionaryInstaller() { + // static installer only + } + + /** + * Downloads a dictionary archive and unpacks it. + * + * @param archive The archive location, a gzip-compressed tar. Must not be + * {@code null}. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if fetching, reading, or writing fails, or the archive + * contains no dictionary file. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static int install(URI archive, Path targetDirectory) throws IOException { + if (archive == null || targetDirectory == null) { + throw new IllegalArgumentException("archive and targetDirectory must not be null"); + } + try (InputStream in = archive.toURL().openStream()) { + return extract(in, targetDirectory); + } + } + + /** + * Unpacks a dictionary archive stream. + * + * @param archiveStream The gzip-compressed tar content. Must not be {@code null}. + * Not closed. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if reading or writing fails, or the archive contains no + * dictionary file. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static int extract(InputStream archiveStream, Path targetDirectory) + throws IOException { + if (archiveStream == null || targetDirectory == null) { + throw new IllegalArgumentException("stream and targetDirectory must not be null"); + } + Files.createDirectories(targetDirectory); + final InputStream tar = new GZIPInputStream(archiveStream); + final byte[] header = new byte[TAR_BLOCK]; + int extracted = 0; + while (readBlock(tar, header)) { + if (isEndBlock(header)) { + break; + } + final String name = headerName(header); + final long size = headerSize(header); + final char type = (char) header[TAR_TYPE_OFFSET]; + final String baseName = baseName(name); + final boolean wanted = (type == '0' || type == 0) + && (baseName.endsWith(".csv") || baseName.endsWith(".def") + || "dicrc".equals(baseName)); + if (wanted) { + final Path file = targetDirectory.resolve(baseName); + try (InputStream entry = boundedStream(tar, size)) { + Files.copy(entry, file, StandardCopyOption.REPLACE_EXISTING); + } + extracted++; + skip(tar, padding(size)); + } else { + skip(tar, size + padding(size)); + } + } + if (extracted == 0) { + throw new IOException("the archive contains no dictionary file"); + } + return extracted; + } + + private static boolean readBlock(InputStream in, byte[] block) throws IOException { + int filled = 0; + while (filled < block.length) { + final int read = in.read(block, filled, block.length - filled); + if (read < 0) { + if (filled == 0) { + return false; + } + throw new IOException("truncated tar header"); + } + filled += read; + } + return true; + } + + private static boolean isEndBlock(byte[] block) { + for (final byte b : block) { + if (b != 0) { + return false; + } + } + return true; + } + + private static String headerName(byte[] header) { + int end = 0; + while (end < TAR_NAME_LENGTH && header[end] != 0) { + end++; + } + return new String(header, 0, end, StandardCharsets.UTF_8); + } + + private static long headerSize(byte[] header) throws IOException { + long size = 0; + for (int i = TAR_SIZE_OFFSET; i < TAR_SIZE_OFFSET + TAR_SIZE_LENGTH; i++) { + final byte b = header[i]; + if (b == 0 || b == ' ') { + continue; + } + if (b < '0' || b > '7') { + throw new IOException("malformed tar size field"); + } + size = size * 8 + (b - '0'); + } + return size; + } + + private static String baseName(String name) { + final int slash = name.lastIndexOf('/'); + return slash < 0 ? name : name.substring(slash + 1); + } + + private static long padding(long size) { + final long remainder = size % TAR_BLOCK; + return remainder == 0 ? 0 : TAR_BLOCK - remainder; + } + + private static void skip(InputStream in, long bytes) throws IOException { + long remaining = bytes; + final byte[] buffer = new byte[8192]; + while (remaining > 0) { + final int read = in.read(buffer, 0, (int) Math.min(buffer.length, remaining)); + if (read < 0) { + throw new IOException("truncated tar entry"); + } + remaining -= read; + } + } + + /** Wraps the tar stream so exactly one entry's bytes are readable. */ + private static InputStream boundedStream(InputStream in, long size) { + return new InputStream() { + private long remaining = size; + + @Override + public int read() throws IOException { + if (remaining <= 0) { + return -1; + } + final int b = in.read(); + if (b < 0) { + throw new IOException("truncated tar entry"); + } + remaining--; + return b; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + if (remaining <= 0) { + return -1; + } + final int read = in.read(buffer, offset, (int) Math.min(length, remaining)); + if (read < 0) { + throw new IOException("truncated tar entry"); + } + remaining -= read; + return read; + } + }; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java new file mode 100644 index 0000000000..a6694ddc58 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.util.List; + +import opennlp.tools.util.Span; + +/** + * One morpheme from lattice segmentation: the {@link Span} it covers in the original + * text, its surface form, and the feature columns its dictionary entry carries. + * + *

The features are the entry's columns exactly as listed in the dictionary, since + * different dictionaries carry different schemas: part of speech first by convention, + * then dictionary-specific columns such as conjugation, base form, or reading. A + * morpheme produced by unknown-word handling has the unknown entry's features and is + * marked as such.

+ * + * @param span The location of the morpheme in the original text. Must not be + * {@code null}. + * @param surface The covered text. Must not be {@code null} or empty. + * @param features The dictionary feature columns. Must not be {@code null}. + * @param unknown Whether the morpheme came from unknown-word handling rather than a + * lexicon entry. + * + * @since 3.0.0 + */ +public record Morpheme(Span span, String surface, List features, + boolean unknown) { + + /** + * Validates the morpheme. + * + * @throws IllegalArgumentException Thrown if {@code span}, {@code surface}, or + * {@code features} is {@code null}, or {@code surface} is empty. + */ + public Morpheme { + if (span == null) { + throw new IllegalArgumentException("span must not be null"); + } + if (surface == null || surface.isEmpty()) { + throw new IllegalArgumentException("surface must not be null or empty"); + } + if (features == null) { + throw new IllegalArgumentException("features must not be null"); + } + features = List.copyOf(features); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java new file mode 100644 index 0000000000..1c3f40a8d3 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.Span; + +/** + * Tests the lattice segmenter against a project-authored miniature dictionary; no + * external dictionary data is involved. + */ +public class LatticeTokenizerTest { + + @TempDir + static Path directory; + + private static LatticeTokenizer tokenizer; + + @BeforeAll + static void loadDictionary() throws IOException { + write("lexicon.csv", String.join("\n", + "東京,0,0,3000,noun,proper", + "京都,0,0,3000,noun,proper", + "東,0,0,6000,noun,common", + "都,0,0,4000,noun,suffix", + "に,0,0,1000,particle,case", + "行く,0,0,3000,verb,base", + "")); + write("matrix.def", "1 1\n0 0 0\n"); + write("char.def", String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "LATIN 1 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "0x0041..0x005A LATIN", + "0x0061..0x007A LATIN", + "")); + write("unk.def", String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "LATIN,0,0,4000,noun,foreign", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "")); + tokenizer = new LatticeTokenizer(MecabDictionary.load(directory)); + } + + private static void write(String name, String content) throws IOException { + Files.write(directory.resolve(name), content.getBytes(StandardCharsets.UTF_8)); + } + + @Test + void testLatticePrefersTheCheaperSegmentation() { + // the famous case: Tokyo+Metro must win over East+Kyoto + final String text = "東京都に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "都", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + } + + @Test + void testMorphemesCarryDictionaryFeatures() { + final List morphemes = + tokenizer.analyze("東京都に行く"); + Assertions.assertEquals(4, morphemes.size()); + Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); + Assertions.assertEquals(List.of("particle", "case"), morphemes.get(2).features()); + Assertions.assertEquals(false, morphemes.get(0).unknown()); + } + + @Test + void testUnknownLatinRunGroupsIntoOneMorpheme() { + final List morphemes = tokenizer.analyze("ABCに行く"); + Assertions.assertEquals(3, morphemes.size()); + Assertions.assertEquals("ABC", morphemes.get(0).surface()); + Assertions.assertEquals(true, morphemes.get(0).unknown()); + Assertions.assertEquals(List.of("noun", "foreign"), morphemes.get(0).features()); + } + + @Test + void testUnknownKanjiPreferOneMorphemeOverTwo() { + final List morphemes = tokenizer.analyze("峠道に行く"); + Assertions.assertEquals(3, morphemes.size()); + Assertions.assertEquals("峠道", morphemes.get(0).surface()); + Assertions.assertEquals(true, morphemes.get(0).unknown()); + } + + @Test + void testWhitespaceSeparatesAndIsNeverAMorpheme() { + final String text = "東京 に 行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(3, 4), new Span(5, 7)}, + tokenizer.tokenizePos(text)); + Assertions.assertEquals(0, tokenizer.analyze(" ").size()); + Assertions.assertEquals(0, tokenizer.analyze("").size()); + } + + @Test + void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + + Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("char.def"), + "KANJI 0 0 2\n0x4E00..0x9FFF KANJI\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("unk.def"), + "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LatticeTokenizer(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionary.load(null)); + Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.analyze(null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java new file mode 100644 index 0000000000..fe999e4609 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.GZIPOutputStream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class MecabDictionaryInstallerTest { + + /** Writes one ustar entry: a 512-byte header block and padded content. */ + private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) + throws IOException { + final byte[] header = new byte[512]; + final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(mode, 0, header, 100, mode.length); + final String size = String.format("%011o", content.length); + System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); + header[156] = '0'; + for (int i = 148; i < 156; i++) { + header[i] = ' '; + } + int checksum = 0; + for (final byte b : header) { + checksum += b & 0xFF; + } + final String checksumText = String.format("%06o", checksum); + System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); + header[154] = 0; + header[155] = ' '; + tar.write(header); + tar.write(content); + final int padding = (512 - content.length % 512) % 512; + tar.write(new byte[padding]); + } + + private static byte[] archive(String[][] entries) throws IOException { + final ByteArrayOutputStream tar = new ByteArrayOutputStream(); + for (final String[] entry : entries) { + tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); + } + tar.write(new byte[1024]); + final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { + gzip.write(tar.toByteArray()); + } + return compressed.toByteArray(); + } + + @Test + void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target) + throws IOException { + final byte[] archive = archive(new String[][] { + {"dict-1.0/lexicon.csv", "cat,0,0,100,noun\n"}, + {"dict-1.0/matrix.def", "1 1\n0 0 0\n"}, + {"dict-1.0/char.def", "DEFAULT 0 1 0\n"}, + {"dict-1.0/unk.def", "DEFAULT,0,0,10000,unknown\n"}, + {"dict-1.0/README", "not a dictionary file"}, + {"dict-1.0/dicrc", "config"}}); + + final int extracted = MecabDictionaryInstaller.extract( + new ByteArrayInputStream(archive), target); + + Assertions.assertEquals(5, extracted); + Assertions.assertTrue(Files.exists(target.resolve("lexicon.csv"))); + Assertions.assertTrue(Files.exists(target.resolve("matrix.def"))); + Assertions.assertTrue(Files.exists(target.resolve("char.def"))); + Assertions.assertTrue(Files.exists(target.resolve("unk.def"))); + Assertions.assertTrue(Files.exists(target.resolve("dicrc"))); + Assertions.assertTrue(Files.notExists(target.resolve("README"))); + Assertions.assertEquals("cat,0,0,100,noun\n", + Files.readString(target.resolve("lexicon.csv"))); + } + + @Test + void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target) + throws IOException { + final Path archiveFile = source.resolve("dict.tar.gz"); + Files.write(archiveFile, archive(new String[][] { + {"d/words.csv", "cat,0,0,100,noun\n"}, + {"d/matrix.def", "1 1\n0 0 0\n"}})); + + final int extracted = + MecabDictionaryInstaller.install(archiveFile.toUri(), target); + + Assertions.assertEquals(2, extracted); + Assertions.assertTrue(Files.exists(target.resolve("words.csv"))); + } + + @Test + void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target) + throws IOException { + final byte[] archive = archive(new String[][] {{"readme.txt", "nothing here"}}); + Assertions.assertThrows(IOException.class, () -> MecabDictionaryInstaller.extract( + new ByteArrayInputStream(archive), target)); + } + + @Test + void testInvalidArguments(@TempDir Path target) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.install(null, target)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.extract(null, target)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.extract( + new ByteArrayInputStream(new byte[0]), null)); + } +} From 67d7643c6ce6701bfba2199a6e2acbdd0959b10c Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 13:56:19 -0400 Subject: [PATCH 02/14] OPENNLP-1894: Character trie for lattice prefix search Common-prefix lookup walks a trie built at load time instead of probing substrings per length, terminating on the first missing prefix and allocating nothing per position. (cherry picked from commit e10ce4b2b8d47821f4ffffcc1da109dcdb7219ba) --- .../tokenize/lattice/LatticeTokenizer.java | 15 ++-- .../tokenize/lattice/MecabDictionary.java | 68 +++++++++++++++++-- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index c022374a5e..c811fd3266 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -203,19 +203,14 @@ private void relax(Node candidate, List predecessors) { private List candidates(String text, int from, int to, int offset) { final int position = from + offset; final List candidates = new ArrayList<>(); - final int longest = Math.min(dictionary.maxSurfaceLength(), to - position); - boolean lexiconMatch = false; - for (int length = 1; length <= longest; length++) { - final List entries = - dictionary.lookup(text.substring(position, position + length)); - if (entries == null) { - continue; - } - lexiconMatch = true; + final boolean[] matched = new boolean[1]; + dictionary.prefixMatches(text, position, to, (length, entries) -> { + matched[0] = true; for (final WordEntry entry : entries) { candidates.add(new Node(position, position + length, entry, false)); } - } + }); + final boolean lexiconMatch = matched[0]; final Category category = dictionary.categoryOf(text.charAt(position)); if (!lexiconMatch || category.invoke()) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 5e1c60f1a6..2e20b8c2d5 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -57,7 +57,25 @@ record WordEntry(int leftId, int rightId, int cost, List features) { record Category(String name, boolean invoke, boolean group, int length) { } - private final Map> lexicon; + /** One node of the lexicon trie, keyed by the next surface character. */ + private static final class TrieNode { + private final Map children = new HashMap<>(); + private List entries; + } + + /** Receives one common-prefix match during {@link #prefixMatches}. */ + interface PrefixMatchConsumer { + + /** + * Accepts one match. + * + * @param length The matched surface length in characters. + * @param entries The lexicon entries for that surface. + */ + void accept(int length, List entries); + } + + private final TrieNode lexicon; private final int maxSurfaceLength; private final short[] connectionCosts; private final int rightSize; @@ -65,7 +83,7 @@ record Category(String name, boolean invoke, boolean group, int length) { private final String[] categoryOfChar; private final Map> unknownEntries; - private MecabDictionary(Map> lexicon, int maxSurfaceLength, + private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, short[] connectionCosts, int rightSize, Map categories, String[] categoryOfChar, Map> unknownEntries) { this.lexicon = lexicon; @@ -150,8 +168,22 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc final Map> unknown = new HashMap<>(); readLexicon(directory.resolve("unk.def"), charset, unknown); - return new MecabDictionary(lexicon, maxSurface, costs, rightSize, categories, - categoryOfChar, unknown); + return new MecabDictionary(buildTrie(lexicon), maxSurface, costs, rightSize, + categories, categoryOfChar, unknown); + } + + /** Folds the surface-keyed lexicon into a character trie for prefix search. */ + private static TrieNode buildTrie(Map> lexicon) { + final TrieNode root = new TrieNode(); + for (final Map.Entry> entry : lexicon.entrySet()) { + TrieNode node = root; + final String surface = entry.getKey(); + for (int i = 0; i < surface.length(); i++) { + node = node.children.computeIfAbsent(surface.charAt(i), key -> new TrieNode()); + } + node.entries = List.copyOf(entry.getValue()); + } + return root; } /** Reads one lexicon-format CSV file; returns the longest surface seen. */ @@ -232,7 +264,33 @@ private static void readCharacterDefinition(Path file, Charset charset, * @return The entries, or {@code null} when the surface is not listed. */ List lookup(String surface) { - return lexicon.get(surface); + TrieNode node = lexicon; + for (int i = 0; i < surface.length() && node != null; i++) { + node = node.children.get(surface.charAt(i)); + } + return node == null ? null : node.entries; + } + + /** + * Reports every lexicon surface starting at a text position, walking the trie once + * with no substring allocation. + * + * @param text The text being segmented. + * @param from The position surfaces must start at. + * @param to The exclusive end of the searchable stretch. + * @param consumer Receives each match. + */ + void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) { + TrieNode node = lexicon; + for (int i = from; i < to; i++) { + node = node.children.get(text.charAt(i)); + if (node == null) { + return; + } + if (node.entries != null) { + consumer.accept(i - from + 1, node.entries); + } + } } /** @return The longest surface form in the lexicon, bounding prefix enumeration. */ From 376e419b6c821583765ef21d24eb57c8f9401555 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 13:58:23 -0400 Subject: [PATCH 03/14] OPENNLP-1894: Frequency-driven segmentation over user-supplied lexicons A Viterbi search maximizing summed word log-probabilities segments Chinese and similar scripts from a plain word-count lexicon, with unlisted characters falling back to single-character words. The user supplies the lexicon and thereby accepts its license; nothing is bundled. (cherry picked from commit bff3f23d4858562171940d09ba2c2321f5005ab0) --- .../tokenize/lattice/UnigramSegmenter.java | 258 ++++++++++++++++++ .../lattice/UnigramSegmenterTest.java | 108 ++++++++ 2 files changed, 366 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java new file mode 100644 index 0000000000..3d1ca9884e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; + +/** + * Frequency-driven segmentation for Chinese and similar scripts: a Viterbi search that + * maximizes the summed log-probability of the words in a user-supplied frequency + * lexicon, with unlisted characters falling back to single-character words. This is the + * unigram model behind common Chinese segmenters; it carries no connection costs, so it + * is lighter than the {@link LatticeTokenizer} and fits lexicons that list only words + * and counts. + * + *

The lexicon format is one entry per line: the word, its count, and optionally a + * tag, separated by whitespace. The user supplies the lexicon file and thereby accepts + * its license; nothing is bundled. Every reported span is in original text + * coordinates.

+ * + *

Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +public class UnigramSegmenter implements Tokenizer { + + /** The log-probability charged to a character the lexicon does not know. */ + private final double unknownLogProbability; + + private final WordTrie trie; + + /** A minimal trie over words with their log-probabilities. */ + private static final class WordTrie { + private final Map children = new HashMap<>(); + private double logProbability = Double.NaN; + } + + private UnigramSegmenter(WordTrie trie, double unknownLogProbability) { + this.trie = trie; + this.unknownLogProbability = unknownLogProbability; + } + + /** + * Loads a frequency lexicon encoded in UTF-8. + * + * @param lexicon The lexicon file. Must not be {@code null}. + * @return The segmenter. Never {@code null}. + * @throws IOException Thrown if reading fails or the lexicon is empty or malformed. + * @throws IllegalArgumentException Thrown if {@code lexicon} is {@code null}. + */ + public static UnigramSegmenter load(Path lexicon) throws IOException { + return load(lexicon, StandardCharsets.UTF_8); + } + + /** + * Loads a frequency lexicon. + * + * @param lexicon The lexicon file: one word, its count, and an optional tag per + * line. Must not be {@code null}. + * @param charset The lexicon encoding. Must not be {@code null}. + * @return The segmenter. Never {@code null}. + * @throws IOException Thrown if reading fails or the lexicon is empty or malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static UnigramSegmenter load(Path lexicon, Charset charset) throws IOException { + if (lexicon == null || charset == null) { + throw new IllegalArgumentException("lexicon and charset must not be null"); + } + try (InputStream in = Files.newInputStream(lexicon)) { + return load(in, charset); + } + } + + /** + * Loads a frequency lexicon from a stream. + * + * @param lexiconStream The lexicon content. Must not be {@code null}. Not closed. + * @param charset The lexicon encoding. Must not be {@code null}. + * @return The segmenter. Never {@code null}. + * @throws IOException Thrown if reading fails or the lexicon is empty or malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) + throws IOException { + if (lexiconStream == null || charset == null) { + throw new IllegalArgumentException("stream and charset must not be null"); + } + final Map counts = new HashMap<>(); + long total = 0; + final String content = new String(lexiconStream.readAllBytes(), charset); + int lineStart = 0; + int lineNumber = 0; + for (int i = 0; i <= content.length(); i++) { + if (i < content.length() && content.charAt(i) != '\n') { + continue; + } + lineNumber++; + final String line = content.substring(lineStart, i).trim(); + lineStart = i + 1; + if (line.isEmpty()) { + continue; + } + final int wordEnd = whitespaceIndex(line); + if (wordEnd < 0) { + throw new IOException("lexicon line " + lineNumber + " has no count"); + } + final String word = line.substring(0, wordEnd); + int countStart = wordEnd; + while (countStart < line.length() && StringUtil.isWhitespace(line.charAt(countStart))) { + countStart++; + } + int countEnd = countStart; + while (countEnd < line.length() && !StringUtil.isWhitespace(line.charAt(countEnd))) { + countEnd++; + } + final long count; + try { + count = Long.parseLong(line.substring(countStart, countEnd)); + } catch (NumberFormatException e) { + throw new IOException("malformed count at lexicon line " + lineNumber, e); + } + if (count <= 0) { + throw new IOException("count must be positive at lexicon line " + lineNumber); + } + counts.merge(word, count, Long::sum); + total += count; + } + if (counts.isEmpty()) { + throw new IOException("the lexicon lists no words"); + } + + final WordTrie root = new WordTrie(); + final double logTotal = Math.log(total); + for (final Map.Entry entry : counts.entrySet()) { + WordTrie node = root; + final String word = entry.getKey(); + for (int c = 0; c < word.length(); c++) { + node = node.children.computeIfAbsent(word.charAt(c), + key -> new WordTrie()); + } + node.logProbability = Math.log(entry.getValue()) - logTotal; + } + // rarer than any listed word: a fraction of a single count + final double unknown = Math.log(0.5) - logTotal; + return new UnigramSegmenter(root, unknown); + } + + @Override + public String[] tokenize(String s) { + final Span[] spans = tokenizePos(s); + final String[] tokens = new String[spans.length]; + for (int i = 0; i < tokens.length; i++) { + tokens[i] = s.substring(spans[i].getStart(), spans[i].getEnd()); + } + return tokens; + } + + @Override + public Span[] tokenizePos(String s) { + if (s == null) { + throw new IllegalArgumentException("text must not be null"); + } + final List spans = new ArrayList<>(); + int start = 0; + while (start < s.length()) { + if (StringUtil.isWhitespace(s.charAt(start))) { + start++; + continue; + } + int end = start; + while (end < s.length() && !StringUtil.isWhitespace(s.charAt(end))) { + end++; + } + decode(s, start, end, spans); + start = end; + } + return spans.toArray(new Span[0]); + } + + /** Viterbi over word log-probabilities within one whitespace-free stretch. */ + private void decode(String text, int from, int to, List spans) { + final int length = to - from; + final double[] best = new double[length + 1]; + final int[] previous = new int[length + 1]; + for (int i = 1; i <= length; i++) { + best[i] = Double.NEGATIVE_INFINITY; + } + for (int i = 0; i < length; i++) { + if (best[i] == Double.NEGATIVE_INFINITY) { + continue; + } + // the single-character fallback keeps every position reachable + final double fallback = best[i] + unknownLogProbability; + if (fallback > best[i + 1]) { + best[i + 1] = fallback; + previous[i + 1] = i; + } + WordTrie node = trie; + for (int j = from + i; j < to; j++) { + node = node.children.get(text.charAt(j)); + if (node == null) { + break; + } + if (!Double.isNaN(node.logProbability)) { + final int end = j - from + 1; + final double score = best[i] + node.logProbability; + if (score > best[end]) { + best[end] = score; + previous[end] = i; + } + } + } + } + final List reversed = new ArrayList<>(); + for (int end = length; end > 0; end = previous[end]) { + reversed.add(new Span(from + previous[end], from + end)); + } + for (int i = reversed.size() - 1; i >= 0; i--) { + spans.add(reversed.get(i)); + } + } + + private static int whitespaceIndex(String text) { + for (int i = 0; i < text.length(); i++) { + if (StringUtil.isWhitespace(text.charAt(i))) { + return i; + } + } + return -1; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java new file mode 100644 index 0000000000..a6104c1958 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +/** + * Tests the frequency-driven segmenter against a project-authored miniature lexicon; + * no external lexicon data is involved. + */ +public class UnigramSegmenterTest { + + private static final String LEXICON = String.join("\n", + "我 5000 r", + "来到 2000 v", + "北京 3000 ns", + "清华大学 800 nt", + "清华 400 ns", + "华大 100 ns", + "大学 1500 n", + "的 9000 uj", + ""); + + private static UnigramSegmenter segmenter; + + @BeforeAll + static void loadLexicon() throws IOException { + segmenter = UnigramSegmenter.load( + new ByteArrayInputStream(LEXICON.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + } + + @Test + void testPrefersWholeWordsOverFragments() { + Assertions.assertArrayEquals( + new String[] {"我", "来到", "北京", "清华大学"}, + segmenter.tokenize("我来到北京清华大学")); + } + + @Test + void testSpansStayInOriginalCoordinates() { + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 9)}, + segmenter.tokenizePos("我来到北京清华大学")); + } + + @Test + void testUnknownCharactersFallBackToSingles() { + Assertions.assertArrayEquals( + new String[] {"我", "爱", "北京"}, + segmenter.tokenize("我爱北京")); + } + + @Test + void testWhitespaceSeparates() { + Assertions.assertArrayEquals( + new String[] {"北京", "大学"}, + segmenter.tokenize("北京 大学")); + Assertions.assertEquals(0, segmenter.tokenizePos(" ").length); + } + + @Test + void testMalformedLexiconsFailLoud() { + Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( + new ByteArrayInputStream("word\n".getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( + new ByteArrayInputStream("word abc\n".getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( + new ByteArrayInputStream("word 0\n".getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( + new ByteArrayInputStream("\n\n".getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((java.nio.file.Path) null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> segmenter.tokenizePos(null)); + } +} From b964966626961f688296cfbf30ddadd9e520122a Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:20:06 -0400 Subject: [PATCH 04/14] OPENNLP-1894: Usage example and edge-case tests for the lattice and unigram segmenters, corrected javadoc --- .../tokenize/lattice/LatticeTokenizer.java | 6 +- .../tokenize/lattice/MecabDictionary.java | 48 ++++- .../lattice/MecabDictionaryInstaller.java | 57 ++++- .../tokenize/lattice/UnigramSegmenter.java | 13 +- .../lattice/LatticeTokenizerTest.java | 149 +++++++++++++ .../lattice/LatticeUsageExampleTest.java | 198 ++++++++++++++++++ .../lattice/UnigramSegmenterTest.java | 67 ++++++ 7 files changed, 529 insertions(+), 9 deletions(-) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index c811fd3266..cb610f74fe 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -225,7 +225,8 @@ private List candidates(String text, int from, int to, int offset) { } } if (candidates.isEmpty()) { - // no entry and no template: a single-character fallback keeps the lattice alive + // Neither the lexicon nor the character's category produced a candidate here, so a + // single-character entry from the DEFAULT template keeps the lattice connected. final List fallback = dictionary.unknownEntries("DEFAULT"); if (fallback != null) { for (final WordEntry entry : fallback) { @@ -251,7 +252,8 @@ private static void addUnknown(List candidates, int position, int runEnd, final int lengths = category.length(); for (int length = 1; length <= lengths && position + length <= to; length++) { if (category.group() && position + length == runEnd) { - continue; // already emitted as the grouped run + // This length coincides with the grouped run emitted above; skip the duplicate. + continue; } for (final WordEntry entry : templates) { candidates.add(new Node(position, position + length, entry, true)); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 2e20b8c2d5..908bc75f2c 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -293,7 +293,7 @@ void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) } } - /** @return The longest surface form in the lexicon, bounding prefix enumeration. */ + /** @return The length in characters of the longest surface form in the lexicon. */ int maxSurfaceLength() { return maxSurfaceLength; } @@ -331,6 +331,14 @@ List unknownEntries(String category) { return unknownEntries.get(category); } + /** + * Reads a required dictionary file into its lines, accepting LF and CRLF endings. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @return The lines without their line terminators. Never {@code null}. + * @throws IOException Thrown if the file is missing or reading fails. + */ private static List readLines(Path file, Charset charset) throws IOException { if (!Files.exists(file)) { throw new IOException("required dictionary file is missing: " + file); @@ -351,11 +359,25 @@ private static List readLines(Path file, Charset charset) throws IOExcep return lines; } + /** + * Removes a trailing {@code #} comment from a {@code char.def} line. + * + * @param line The raw line. + * @return The line up to but excluding the first {@code #}, or the whole line when + * there is none. + */ private static String stripComment(String line) { final int hash = line.indexOf('#'); return hash < 0 ? line : line.substring(0, hash); } + /** + * Splits a lexicon line at every comma. Surfaces containing commas are not + * representable, matching the plain-text lexicon format. + * + * @param line The line to split. + * @return The fields in order, empty fields included. Never {@code null}. + */ private static List splitCsv(String line) { final List fields = new ArrayList<>(); int start = 0; @@ -368,6 +390,12 @@ private static List splitCsv(String line) { return fields; } + /** + * Splits a line into its whitespace-separated fields. + * + * @param line The line to split. + * @return The non-empty fields in order. Never {@code null}. + */ private static String[] splitWhitespace(String line) { final List parts = new ArrayList<>(); int start = -1; @@ -384,6 +412,15 @@ private static String[] splitWhitespace(String line) { return parts.toArray(new String[0]); } + /** + * Parses a decimal integer field, reporting the file and line on failure. + * + * @param text The field text. + * @param file The file being read, for the error message. + * @param lineNumber The line being read, for the error message. + * @return The parsed value. + * @throws IOException Thrown if the field is not a valid integer. + */ private static int parseInt(String text, String file, int lineNumber) throws IOException { try { @@ -393,6 +430,15 @@ private static int parseInt(String text, String file, int lineNumber) } } + /** + * Parses a {@code 0x}-prefixed hexadecimal code point from {@code char.def}. + * + * @param text The field text including the {@code 0x} prefix. + * @param file The file being read, for the error message. + * @param lineNumber The line being read, for the error message. + * @return The parsed code point. + * @throws IOException Thrown if the field is not a valid hexadecimal code point. + */ private static int parseCodePoint(String text, Path file, int lineNumber) throws IOException { try { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 33f3ef58dd..9bbbde7902 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -34,9 +34,11 @@ * built in. * *

The installer reads gzip-compressed tar archives, the format the common - * distributions use, and extracts only the files a {@link MecabDictionary} reads: - * {@code *.csv}, {@code *.def}, and {@code dicrc}. Entries are flattened to their base - * names, which also means no archive path can escape the target directory.

+ * distributions use, and extracts only the dictionary payload: the {@code *.csv} + * lexicon files and {@code *.def} definition files that a {@link MecabDictionary} + * reads, plus the {@code dicrc} configuration file distributions ship alongside them. + * Entries are flattened to their base names, which also means no archive path can + * escape the target directory.

* * @since 3.0.0 */ @@ -49,7 +51,7 @@ public final class MecabDictionaryInstaller { private static final int TAR_TYPE_OFFSET = 156; private MecabDictionaryInstaller() { - // static installer only + // This class exposes only static methods and is never instantiated. } /** @@ -122,6 +124,15 @@ public static int extract(InputStream archiveStream, Path targetDirectory) return extracted; } + /** + * Fills one tar block from the stream. + * + * @param in The tar stream. + * @param block The block buffer to fill completely. + * @return {@code true} when a full block was read, {@code false} at a clean end of + * stream before any byte of the block. + * @throws IOException Thrown if the stream ends inside the block or reading fails. + */ private static boolean readBlock(InputStream in, byte[] block) throws IOException { int filled = 0; while (filled < block.length) { @@ -137,6 +148,12 @@ private static boolean readBlock(InputStream in, byte[] block) throws IOExceptio return true; } + /** + * Recognizes the all-zero block that terminates a tar archive. + * + * @param block The block to inspect. + * @return {@code true} when every byte is zero. + */ private static boolean isEndBlock(byte[] block) { for (final byte b : block) { if (b != 0) { @@ -146,6 +163,12 @@ private static boolean isEndBlock(byte[] block) { return true; } + /** + * Reads the NUL-terminated entry name from a tar header block. + * + * @param header The header block. + * @return The entry name. Never {@code null}. + */ private static String headerName(byte[] header) { int end = 0; while (end < TAR_NAME_LENGTH && header[end] != 0) { @@ -154,6 +177,13 @@ private static String headerName(byte[] header) { return new String(header, 0, end, StandardCharsets.UTF_8); } + /** + * Reads the octal entry size from a tar header block. + * + * @param header The header block. + * @return The entry size in bytes. + * @throws IOException Thrown if the size field holds a non-octal digit. + */ private static long headerSize(byte[] header) throws IOException { long size = 0; for (int i = TAR_SIZE_OFFSET; i < TAR_SIZE_OFFSET + TAR_SIZE_LENGTH; i++) { @@ -169,16 +199,35 @@ private static long headerSize(byte[] header) throws IOException { return size; } + /** + * Strips any directory prefix from an archive entry name. + * + * @param name The entry name as stored in the archive. + * @return The part after the last {@code /}, or the whole name when there is none. + */ private static String baseName(String name) { final int slash = name.lastIndexOf('/'); return slash < 0 ? name : name.substring(slash + 1); } + /** + * Computes the padding after an entry: tar content is stored in whole blocks. + * + * @param size The entry size in bytes. + * @return The number of padding bytes up to the next block boundary. + */ private static long padding(long size) { final long remainder = size % TAR_BLOCK; return remainder == 0 ? 0 : TAR_BLOCK - remainder; } + /** + * Consumes and discards an exact number of bytes from the stream. + * + * @param in The stream to read from. + * @param bytes The number of bytes to discard. + * @throws IOException Thrown if the stream ends before that many bytes were read. + */ private static void skip(InputStream in, long bytes) throws IOException { long remaining = bytes; final byte[] buffer = new byte[8192]; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index 3d1ca9884e..ea3d188bc5 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -167,7 +167,8 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) } node.logProbability = Math.log(entry.getValue()) - logTotal; } - // rarer than any listed word: a fraction of a single count + // Charge an unlisted character half of one count out of the total, which makes it + // rarer than any listed word: every listed count is at least one. final double unknown = Math.log(0.5) - logTotal; return new UnigramSegmenter(root, unknown); } @@ -216,7 +217,8 @@ private void decode(String text, int from, int to, List spans) { if (best[i] == Double.NEGATIVE_INFINITY) { continue; } - // the single-character fallback keeps every position reachable + // A single-character step at the unknown log-probability keeps every position + // reachable even where no lexicon word matches. final double fallback = best[i] + unknownLogProbability; if (fallback > best[i + 1]) { best[i + 1] = fallback; @@ -247,6 +249,13 @@ private void decode(String text, int from, int to, List spans) { } } + /** + * Finds the first whitespace character in a lexicon line. + * + * @param text The line to scan. + * @return The index of the first whitespace character, or {@code -1} when the line + * contains none. + */ private static int whitespaceIndex(String text) { for (int i = 0; i < text.length(); i++) { if (StringUtil.isWhitespace(text.charAt(i))) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 1c3f40a8d3..b643786a97 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -128,6 +128,155 @@ void testWhitespaceSeparatesAndIsNeverAMorpheme() { Assertions.assertEquals(0, tokenizer.analyze("").size()); } + /** + * Verifies that empty input yields empty results from every view of the tokenizer. + */ + @Test + void testEmptyInputYieldsEmptyResults() { + Assertions.assertArrayEquals(new String[0], tokenizer.tokenize("")); + Assertions.assertArrayEquals(new Span[0], tokenizer.tokenizePos("")); + } + + /** + * Verifies single-character input for a listed surface and for an unlisted kanji: + * both come back as exactly one morpheme covering {@code [0, 1)}, and only the + * unlisted one is marked unknown. + */ + @Test + void testSingleCharacterInput() { + Assertions.assertArrayEquals(new String[] {"に"}, tokenizer.tokenize("に")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("に")); + Assertions.assertFalse(tokenizer.analyze("に").get(0).unknown()); + + final List unknown = tokenizer.analyze("峠"); + Assertions.assertEquals(1, unknown.size()); + Assertions.assertEquals("峠", unknown.get(0).surface()); + Assertions.assertEquals(new Span(0, 1), unknown.get(0).span()); + Assertions.assertTrue(unknown.get(0).unknown()); + } + + /** + * Verifies input made entirely of characters absent from both the lexicon and the + * {@code char.def} mappings: they fall into the DEFAULT category, whose grouping + * setting joins the whole same-category run into one unknown morpheme carrying the + * DEFAULT template's features. + */ + @Test + void testEntirelyUnknownInputGroupsIntoOneDefaultMorpheme() { + final List morphemes = tokenizer.analyze("①②③"); + Assertions.assertEquals(1, morphemes.size()); + Assertions.assertEquals("①②③", morphemes.get(0).surface()); + Assertions.assertEquals(new Span(0, 3), morphemes.get(0).span()); + Assertions.assertTrue(morphemes.get(0).unknown()); + Assertions.assertEquals(List.of("symbol", "unknown"), morphemes.get(0).features()); + } + + /** + * Verifies a mixed run of known and unknown text: the lexicon words around an + * unmapped character are kept intact, the unmapped character becomes its own + * unknown morpheme, and every span stays in original text coordinates. + */ + @Test + void testMixedKnownAndUnknownRuns() { + final String text = "東京①に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "①", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + final List morphemes = tokenizer.analyze(text); + Assertions.assertFalse(morphemes.get(0).unknown()); + Assertions.assertTrue(morphemes.get(1).unknown()); + Assertions.assertFalse(morphemes.get(2).unknown()); + } + + /** + * Verifies that spans keep original text coordinates when the interesting content + * does not start at position zero because of leading whitespace. + */ + @Test + void testSpansStayOriginalAfterLeadingWhitespace() { + final String text = " 東京都に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "都", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(2, 4), new Span(4, 5), new Span(5, 6), new Span(6, 8)}, + tokenizer.tokenizePos(text)); + } + + /** + * Verifies that a lexicon row with fewer than the four mandatory columns is + * rejected at load time. + */ + @Test + void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a non-numeric cost column in a lexicon row is rejected at load + * time. + */ + @Test + void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a {@code matrix.def} data line with the wrong number of fields is + * rejected at load time. + */ + @Test + void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("matrix.def"), "1 1\n0 0\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a {@code char.def} code point mapping without a category name is + * rejected at load time. + */ + @Test + void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) + throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("char.def"), + "DEFAULT 0 1 0\n0x4E00..0x9FFF\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies the fail-loud path when a loadable dictionary cannot cover the input: the + * {@code unk.def} has no DEFAULT template, so a character with neither a lexicon + * entry nor a category template stops segmentation with an exception instead of + * being dropped silently. + */ + @Test + void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) + throws IOException { + Files.write(partial.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(partial.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(partial.resolve("char.def"), + "DEFAULT 0 1 0\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n" + .getBytes(StandardCharsets.UTF_8)); + Files.write(partial.resolve("unk.def"), + "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); + final LatticeTokenizer limited = + new LatticeTokenizer(MecabDictionary.load(partial)); + Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("①")); + } + @Test void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java new file mode 100644 index 0000000000..a5e2a39d1a --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.zip.GZIPOutputStream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.Span; + +/** + * Demonstrates the intended end-to-end usage of this package with miniature, + * project-authored data: a mecab-format dictionary archive is installed with + * {@link MecabDictionaryInstaller}, loaded as a {@link MecabDictionary}, and segmented + * with a {@link LatticeTokenizer}; a plain frequency lexicon is loaded and segmented + * with a {@link UnigramSegmenter}. Everything is written to a temporary directory by + * the test itself; no external dictionary or lexicon data and no network access are + * involved. + */ +public class LatticeUsageExampleTest { + + /** + * Appends one ustar file entry to a growing tar image: a 512-byte header block + * followed by the content padded to a block boundary. + * + * @param tar The tar image under construction. Must not be {@code null}. + * @param name The entry name including any directory prefix. Must not be + * {@code null}, empty, or longer than the 100-byte header name field. + * @param content The entry content bytes. Must not be {@code null}. + * @throws IOException Thrown if writing to the in-memory stream fails. + * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. + */ + private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) + throws IOException { + final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + if (nameBytes.length == 0 || nameBytes.length > 100) { + throw new IllegalArgumentException( + "entry name must be 1..100 bytes, got " + nameBytes.length); + } + final byte[] header = new byte[512]; + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(mode, 0, header, 100, mode.length); + final String size = String.format("%011o", content.length); + System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); + header[156] = '0'; + for (int i = 148; i < 156; i++) { + header[i] = ' '; + } + int checksum = 0; + for (final byte b : header) { + checksum += b & 0xFF; + } + final String checksumText = String.format("%06o", checksum); + System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); + header[154] = 0; + header[155] = ' '; + tar.write(header); + tar.write(content); + final int padding = (512 - content.length % 512) % 512; + tar.write(new byte[padding]); + } + + /** + * Builds a gzip-compressed tar archive from name and content pairs, the layout a + * dictionary distribution ships in. + * + * @param entries The entries as {@code {name, content}} pairs. Must not be + * {@code null}. + * @return The compressed archive bytes. Never {@code null}. + * @throws IOException Thrown if writing to the in-memory streams fails. + */ + private static byte[] gzippedTar(String[][] entries) throws IOException { + final ByteArrayOutputStream tar = new ByteArrayOutputStream(); + for (final String[] entry : entries) { + tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); + } + tar.write(new byte[1024]); + final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { + gzip.write(tar.toByteArray()); + } + return compressed.toByteArray(); + } + + /** + * Walks the full mecab-format flow: package a miniature Japanese dictionary as a + * {@code tar.gz} archive, install it from a file URI, load it, and tokenize. The + * segmentation must pick the cheaper path (Tokyo plus the metropolis suffix) over + * the competing reading (east plus Kyoto), the spans must be in original text + * coordinates, and the morphemes must carry the dictionary's feature columns. + */ + @Test + void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) + throws IOException { + // A minimal but complete dictionary: one lexicon file plus the three definition + // files every mecab-format distribution contains, wrapped like a release archive. + final byte[] archive = gzippedTar(new String[][] { + {"mini-dict-0.1/lexicon.csv", String.join("\n", + "東京,0,0,3000,noun,proper", + "京都,0,0,3000,noun,proper", + "東,0,0,6000,noun,common", + "都,0,0,4000,noun,suffix", + "に,0,0,1000,particle,case", + "行く,0,0,3000,verb,base", + "")}, + {"mini-dict-0.1/matrix.def", "1 1\n0 0 0\n"}, + {"mini-dict-0.1/char.def", String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "")}, + {"mini-dict-0.1/unk.def", String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "")}, + {"mini-dict-0.1/README", "not a dictionary payload file"}}); + final Path archiveFile = work.resolve("mini-dict-0.1.tar.gz"); + Files.write(archiveFile, archive); + + // Install: fetch the archive from the user-chosen location and unpack the payload. + final Path dictionaryDirectory = work.resolve("dictionary"); + final int extracted = + MecabDictionaryInstaller.install(archiveFile.toUri(), dictionaryDirectory); + Assertions.assertEquals(4, extracted); + + // Load and tokenize; both views must agree and stay in original coordinates. + final LatticeTokenizer tokenizer = + new LatticeTokenizer(MecabDictionary.load(dictionaryDirectory)); + final String text = "東京都に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "都", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + + // The analyze view adds the dictionary's feature columns to every morpheme. + final List morphemes = tokenizer.analyze(text); + Assertions.assertEquals(4, morphemes.size()); + Assertions.assertEquals("東京", morphemes.get(0).surface()); + Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); + Assertions.assertFalse(morphemes.get(0).unknown()); + } + + /** + * Walks the frequency-lexicon flow: write a miniature word-count lexicon to a file, + * load it, and segment. The segmentation must recover the listed multi-character + * words with spans in original text coordinates. + */ + @Test + void testLoadAndSegmentWithAFrequencyLexicon(@TempDir Path work) throws IOException { + // One word, its count, and an optional tag per line, whitespace separated. + final Path lexicon = work.resolve("words.txt"); + Files.write(lexicon, String.join("\n", + "我 5000 r", + "来到 2000 v", + "北京 3000 ns", + "天安门 1200 ns", + "").getBytes(StandardCharsets.UTF_8)); + + final UnigramSegmenter segmenter = UnigramSegmenter.load(lexicon); + final String text = "我来到北京天安门"; + Assertions.assertArrayEquals( + new String[] {"我", "来到", "北京", "天安门"}, + segmenter.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 8)}, + segmenter.tokenizePos(text)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index a6104c1958..f086eafa26 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -82,6 +82,73 @@ void testWhitespaceSeparates() { Assertions.assertEquals(0, segmenter.tokenizePos(" ").length); } + /** + * Verifies that empty input yields empty results from both views of the segmenter. + */ + @Test + void testEmptyInputYieldsEmptyResults() { + Assertions.assertArrayEquals(new String[0], segmenter.tokenize("")); + Assertions.assertArrayEquals(new Span[0], segmenter.tokenizePos("")); + } + + /** + * Verifies single-character input for a listed word and for a character the + * lexicon does not know: both come back as exactly one token covering + * {@code [0, 1)}. + */ + @Test + void testSingleCharacterInput() { + Assertions.assertArrayEquals(new String[] {"我"}, segmenter.tokenize("我")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("我")); + Assertions.assertArrayEquals(new String[] {"爱"}, segmenter.tokenize("爱")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("爱")); + } + + /** + * Verifies input made entirely of characters absent from the lexicon: every + * character becomes its own single-character token, since only the unknown + * fallback is available. + */ + @Test + void testEntirelyUnknownInputFallsBackToSingleCharacters() { + Assertions.assertArrayEquals( + new String[] {"x", "y", "z"}, + segmenter.tokenize("xyz")); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 2), new Span(2, 3)}, + segmenter.tokenizePos("xyz")); + } + + /** + * Verifies a mixed run of known and unknown text inside one whitespace-free + * stretch: the unknown character becomes a single token while the listed words + * around it, including the longest listed compound, stay intact. + */ + @Test + void testMixedKnownAndUnknownRuns() { + Assertions.assertArrayEquals( + new String[] {"我", "爱", "清华大学"}, + segmenter.tokenize("我爱清华大学")); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 2), new Span(2, 6)}, + segmenter.tokenizePos("我爱清华大学")); + } + + /** + * Verifies that spans keep original text coordinates when the content does not + * start at position zero because of leading whitespace. + */ + @Test + void testSpansStayOriginalAfterLeadingWhitespace() { + final String text = " 我来到北京"; + Assertions.assertArrayEquals( + new String[] {"我", "来到", "北京"}, + segmenter.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(2, 3), new Span(3, 5), new Span(5, 7)}, + segmenter.tokenizePos(text)); + } + @Test void testMalformedLexiconsFailLoud() { Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( From 264c5feabce88d6cfbc6c727baf9f029268d1ef7 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:32:44 -0400 Subject: [PATCH 05/14] OPENNLP-1894: Keep lattice test sources ASCII-only via Unicode escapes, add an EUC-JP loading example --- .../lattice/LatticeTokenizerTest.java | 68 ++++++++-------- .../lattice/LatticeUsageExampleTest.java | 77 +++++++++++++++---- .../lattice/UnigramSegmenterTest.java | 52 +++++++------ 3 files changed, 126 insertions(+), 71 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index b643786a97..83b823706c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -33,6 +33,10 @@ /** * Tests the lattice segmenter against a project-authored miniature dictionary; no * external dictionary data is involved. + * + *

Source strings are written as Unicode escapes to keep this file ASCII-only; the + * class works over the same miniature Japanese dictionary as the sibling usage + * example, whose javadoc spells out each fixture word.

*/ public class LatticeTokenizerTest { @@ -44,12 +48,12 @@ public class LatticeTokenizerTest { @BeforeAll static void loadDictionary() throws IOException { write("lexicon.csv", String.join("\n", - "東京,0,0,3000,noun,proper", - "京都,0,0,3000,noun,proper", - "東,0,0,6000,noun,common", - "都,0,0,4000,noun,suffix", - "に,0,0,1000,particle,case", - "行く,0,0,3000,verb,base", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", "")); write("matrix.def", "1 1\n0 0 0\n"); write("char.def", String.join("\n", @@ -79,9 +83,9 @@ private static void write(String name, String content) throws IOException { @Test void testLatticePrefersTheCheaperSegmentation() { // the famous case: Tokyo+Metro must win over East+Kyoto - final String text = "東京都に行く"; + final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "都", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, @@ -91,7 +95,7 @@ void testLatticePrefersTheCheaperSegmentation() { @Test void testMorphemesCarryDictionaryFeatures() { final List morphemes = - tokenizer.analyze("東京都に行く"); + tokenizer.analyze("\u6771\u4EAC\u90FD\u306B\u884C\u304F"); Assertions.assertEquals(4, morphemes.size()); Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); Assertions.assertEquals(List.of("particle", "case"), morphemes.get(2).features()); @@ -100,7 +104,7 @@ void testMorphemesCarryDictionaryFeatures() { @Test void testUnknownLatinRunGroupsIntoOneMorpheme() { - final List morphemes = tokenizer.analyze("ABCに行く"); + final List morphemes = tokenizer.analyze("ABC\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); Assertions.assertEquals("ABC", morphemes.get(0).surface()); Assertions.assertEquals(true, morphemes.get(0).unknown()); @@ -109,17 +113,17 @@ void testUnknownLatinRunGroupsIntoOneMorpheme() { @Test void testUnknownKanjiPreferOneMorphemeOverTwo() { - final List morphemes = tokenizer.analyze("峠道に行く"); + final List morphemes = tokenizer.analyze("\u5CE0\u9053\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); - Assertions.assertEquals("峠道", morphemes.get(0).surface()); + Assertions.assertEquals("\u5CE0\u9053", morphemes.get(0).surface()); Assertions.assertEquals(true, morphemes.get(0).unknown()); } @Test void testWhitespaceSeparatesAndIsNeverAMorpheme() { - final String text = "東京 に 行く"; + final String text = "\u6771\u4EAC \u306B \u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(3, 4), new Span(5, 7)}, @@ -144,13 +148,13 @@ void testEmptyInputYieldsEmptyResults() { */ @Test void testSingleCharacterInput() { - Assertions.assertArrayEquals(new String[] {"に"}, tokenizer.tokenize("に")); - Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("に")); - Assertions.assertFalse(tokenizer.analyze("に").get(0).unknown()); + Assertions.assertArrayEquals(new String[] {"\u306B"}, tokenizer.tokenize("\u306B")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("\u306B")); + Assertions.assertFalse(tokenizer.analyze("\u306B").get(0).unknown()); - final List unknown = tokenizer.analyze("峠"); + final List unknown = tokenizer.analyze("\u5CE0"); Assertions.assertEquals(1, unknown.size()); - Assertions.assertEquals("峠", unknown.get(0).surface()); + Assertions.assertEquals("\u5CE0", unknown.get(0).surface()); Assertions.assertEquals(new Span(0, 1), unknown.get(0).span()); Assertions.assertTrue(unknown.get(0).unknown()); } @@ -163,9 +167,9 @@ void testSingleCharacterInput() { */ @Test void testEntirelyUnknownInputGroupsIntoOneDefaultMorpheme() { - final List morphemes = tokenizer.analyze("①②③"); + final List morphemes = tokenizer.analyze("\u2460\u2461\u2462"); Assertions.assertEquals(1, morphemes.size()); - Assertions.assertEquals("①②③", morphemes.get(0).surface()); + Assertions.assertEquals("\u2460\u2461\u2462", morphemes.get(0).surface()); Assertions.assertEquals(new Span(0, 3), morphemes.get(0).span()); Assertions.assertTrue(morphemes.get(0).unknown()); Assertions.assertEquals(List.of("symbol", "unknown"), morphemes.get(0).features()); @@ -178,9 +182,9 @@ void testEntirelyUnknownInputGroupsIntoOneDefaultMorpheme() { */ @Test void testMixedKnownAndUnknownRuns() { - final String text = "東京①に行く"; + final String text = "\u6771\u4EAC\u2460\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "①", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u2460", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, @@ -197,9 +201,9 @@ void testMixedKnownAndUnknownRuns() { */ @Test void testSpansStayOriginalAfterLeadingWhitespace() { - final String text = " 東京都に行く"; + final String text = " \u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "都", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(2, 4), new Span(4, 5), new Span(5, 6), new Span(6, 8)}, @@ -213,7 +217,7 @@ void testSpansStayOriginalAfterLeadingWhitespace() { @Test void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -224,7 +228,7 @@ void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { @Test void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -235,7 +239,7 @@ void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException @Test void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Files.write(broken.resolve("matrix.def"), "1 1\n0 0\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -248,7 +252,7 @@ void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); Files.write(broken.resolve("char.def"), "DEFAULT 0 1 0\n0x4E00..0x9FFF\n".getBytes(StandardCharsets.UTF_8)); @@ -265,7 +269,7 @@ void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) throws IOException { Files.write(partial.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Files.write(partial.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); Files.write(partial.resolve("char.def"), "DEFAULT 0 1 0\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n" @@ -274,13 +278,13 @@ void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); final LatticeTokenizer limited = new LatticeTokenizer(MecabDictionary.load(partial)); - Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("①")); + Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("\u2460")); } @Test void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java index a5e2a39d1a..a61969c902 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -19,6 +19,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -39,6 +40,13 @@ * with a {@link UnigramSegmenter}. Everything is written to a temporary directory by * the test itself; no external dictionary or lexicon data and no network access are * involved. + * + *

Source strings are written as Unicode escapes to keep this file ASCII-only. The + * Japanese fixture words are Tokyo (U+6771 U+4EAC), Kyoto (U+4EAC U+90FD), east + * (U+6771), the metropolis suffix (U+90FD), the case particle ni (U+306B), and the + * verb iku, to go (U+884C U+304F); the Chinese fixture words are wo, I (U+6211), + * laidao, arrive (U+6765 U+5230), Beijing (U+5317 U+4EAC), and Tiananmen + * (U+5929 U+5B89 U+95E8).

*/ public class LatticeUsageExampleTest { @@ -120,12 +128,12 @@ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) // files every mecab-format distribution contains, wrapped like a release archive. final byte[] archive = gzippedTar(new String[][] { {"mini-dict-0.1/lexicon.csv", String.join("\n", - "東京,0,0,3000,noun,proper", - "京都,0,0,3000,noun,proper", - "東,0,0,6000,noun,common", - "都,0,0,4000,noun,suffix", - "に,0,0,1000,particle,case", - "行く,0,0,3000,verb,base", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", "")}, {"mini-dict-0.1/matrix.def", "1 1\n0 0 0\n"}, {"mini-dict-0.1/char.def", String.join("\n", @@ -154,9 +162,9 @@ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) // Load and tokenize; both views must agree and stay in original coordinates. final LatticeTokenizer tokenizer = new LatticeTokenizer(MecabDictionary.load(dictionaryDirectory)); - final String text = "東京都に行く"; + final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "都", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, @@ -165,7 +173,7 @@ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) // The analyze view adds the dictionary's feature columns to every morpheme. final List morphemes = tokenizer.analyze(text); Assertions.assertEquals(4, morphemes.size()); - Assertions.assertEquals("東京", morphemes.get(0).surface()); + Assertions.assertEquals("\u6771\u4EAC", morphemes.get(0).surface()); Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); Assertions.assertFalse(morphemes.get(0).unknown()); } @@ -180,19 +188,58 @@ void testLoadAndSegmentWithAFrequencyLexicon(@TempDir Path work) throws IOExcept // One word, its count, and an optional tag per line, whitespace separated. final Path lexicon = work.resolve("words.txt"); Files.write(lexicon, String.join("\n", - "我 5000 r", - "来到 2000 v", - "北京 3000 ns", - "天安门 1200 ns", + "\u6211 5000 r", + "\u6765\u5230 2000 v", + "\u5317\u4EAC 3000 ns", + "\u5929\u5B89\u95E8 1200 ns", "").getBytes(StandardCharsets.UTF_8)); final UnigramSegmenter segmenter = UnigramSegmenter.load(lexicon); - final String text = "我来到北京天安门"; + final String text = "\u6211\u6765\u5230\u5317\u4EAC\u5929\u5B89\u95E8"; Assertions.assertArrayEquals( - new String[] {"我", "来到", "北京", "天安门"}, + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC", "\u5929\u5B89\u95E8"}, segmenter.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 8)}, segmenter.tokenizePos(text)); } + + /** + * Walks the non-UTF-8 flow that widely used Japanese distributions require: the same + * miniature dictionary is written to disk encoded in EUC-JP and loaded through the + * charset-taking overload. The segmentation must match the UTF-8 run exactly, which + * shows the encoding is a property of loading, not of tokenization. + */ + @Test + void testLoadAnEucJpEncodedDictionary(@TempDir Path work) throws IOException { + final Charset eucJp = Charset.forName("EUC-JP"); + Files.write(work.resolve("lexicon.csv"), String.join("\n", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", + "").getBytes(eucJp)); + Files.write(work.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(eucJp)); + Files.write(work.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "").getBytes(eucJp)); + Files.write(work.resolve("unk.def"), String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "").getBytes(eucJp)); + + final LatticeTokenizer tokenizer = + new LatticeTokenizer(MecabDictionary.load(work, eucJp)); + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize("\u6771\u4EAC\u90FD\u306B\u884C\u304F")); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index f086eafa26..9725b8c729 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -30,18 +30,22 @@ /** * Tests the frequency-driven segmenter against a project-authored miniature lexicon; * no external lexicon data is involved. + * + *

Source strings are written as Unicode escapes to keep this file ASCII-only; the + * class works over the same miniature Chinese frequency lexicon as the sibling usage + * example, whose javadoc spells out each fixture word.

*/ public class UnigramSegmenterTest { private static final String LEXICON = String.join("\n", - "我 5000 r", - "来到 2000 v", - "北京 3000 ns", - "清华大学 800 nt", - "清华 400 ns", - "华大 100 ns", - "大学 1500 n", - "的 9000 uj", + "\u6211 5000 r", + "\u6765\u5230 2000 v", + "\u5317\u4EAC 3000 ns", + "\u6E05\u534E\u5927\u5B66 800 nt", + "\u6E05\u534E 400 ns", + "\u534E\u5927 100 ns", + "\u5927\u5B66 1500 n", + "\u7684 9000 uj", ""); private static UnigramSegmenter segmenter; @@ -56,29 +60,29 @@ static void loadLexicon() throws IOException { @Test void testPrefersWholeWordsOverFragments() { Assertions.assertArrayEquals( - new String[] {"我", "来到", "北京", "清华大学"}, - segmenter.tokenize("我来到北京清华大学")); + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC", "\u6E05\u534E\u5927\u5B66"}, + segmenter.tokenize("\u6211\u6765\u5230\u5317\u4EAC\u6E05\u534E\u5927\u5B66")); } @Test void testSpansStayInOriginalCoordinates() { Assertions.assertArrayEquals(new Span[] { new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 9)}, - segmenter.tokenizePos("我来到北京清华大学")); + segmenter.tokenizePos("\u6211\u6765\u5230\u5317\u4EAC\u6E05\u534E\u5927\u5B66")); } @Test void testUnknownCharactersFallBackToSingles() { Assertions.assertArrayEquals( - new String[] {"我", "爱", "北京"}, - segmenter.tokenize("我爱北京")); + new String[] {"\u6211", "\u7231", "\u5317\u4EAC"}, + segmenter.tokenize("\u6211\u7231\u5317\u4EAC")); } @Test void testWhitespaceSeparates() { Assertions.assertArrayEquals( - new String[] {"北京", "大学"}, - segmenter.tokenize("北京 大学")); + new String[] {"\u5317\u4EAC", "\u5927\u5B66"}, + segmenter.tokenize("\u5317\u4EAC \u5927\u5B66")); Assertions.assertEquals(0, segmenter.tokenizePos(" ").length); } @@ -98,10 +102,10 @@ void testEmptyInputYieldsEmptyResults() { */ @Test void testSingleCharacterInput() { - Assertions.assertArrayEquals(new String[] {"我"}, segmenter.tokenize("我")); - Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("我")); - Assertions.assertArrayEquals(new String[] {"爱"}, segmenter.tokenize("爱")); - Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("爱")); + Assertions.assertArrayEquals(new String[] {"\u6211"}, segmenter.tokenize("\u6211")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("\u6211")); + Assertions.assertArrayEquals(new String[] {"\u7231"}, segmenter.tokenize("\u7231")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("\u7231")); } /** @@ -127,11 +131,11 @@ void testEntirelyUnknownInputFallsBackToSingleCharacters() { @Test void testMixedKnownAndUnknownRuns() { Assertions.assertArrayEquals( - new String[] {"我", "爱", "清华大学"}, - segmenter.tokenize("我爱清华大学")); + new String[] {"\u6211", "\u7231", "\u6E05\u534E\u5927\u5B66"}, + segmenter.tokenize("\u6211\u7231\u6E05\u534E\u5927\u5B66")); Assertions.assertArrayEquals(new Span[] { new Span(0, 1), new Span(1, 2), new Span(2, 6)}, - segmenter.tokenizePos("我爱清华大学")); + segmenter.tokenizePos("\u6211\u7231\u6E05\u534E\u5927\u5B66")); } /** @@ -140,9 +144,9 @@ void testMixedKnownAndUnknownRuns() { */ @Test void testSpansStayOriginalAfterLeadingWhitespace() { - final String text = " 我来到北京"; + final String text = " \u6211\u6765\u5230\u5317\u4EAC"; Assertions.assertArrayEquals( - new String[] {"我", "来到", "北京"}, + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC"}, segmenter.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(2, 3), new Span(3, 5), new Span(5, 7)}, From 41cb53573da5365f5ce40c7e7243e00921af8658 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:32:44 -0400 Subject: [PATCH 06/14] OPENNLP-1894: Document dictionary acquisition with a checksum-verifying download helper --- .../dev/README-mecab-dictionaries.md | 83 +++++++++++++++++++ .../dev/download-mecab-dictionary.sh | 69 +++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md create mode 100755 opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh diff --git a/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md b/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md new file mode 100644 index 0000000000..c3244ba9c4 --- /dev/null +++ b/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md @@ -0,0 +1,83 @@ + + +# CJK dictionaries for the lattice tokenizer + +The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a mecab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and by using it you accept that dictionary's license. Read the license file inside the archive before use; nothing in this repository grants you those terms. + +## Known mecab-format dictionary projects + +| Dictionary | Language | Archive to download | Encoding of the files | +|---|---|---|---| +| IPADIC 2.7.0 | Japanese | `mecab-ipadic-2.7.0-20070801.tar.gz` from the MeCab project's SourceForge file area (`sourceforge.net/projects/mecab/files/mecab-ipadic/2.7.0-20070801/`) | EUC-JP | +| mecab-ko-dic 2.1.1 | Korean | `mecab-ko-dic-2.1.1-20180720.tar.gz` from the project's Bitbucket downloads page (`bitbucket.org/eunjeon/mecab-ko-dic/downloads/`) | UTF-8 | + +Both archives are gzip-compressed tars, the format `MecabDictionaryInstaller` reads. The encoding column matters: `MecabDictionary.load(Path)` assumes UTF-8, and a dictionary in any other encoding is loaded with the two-argument overload, for example `MecabDictionary.load(dir, Charset.forName("EUC-JP"))` for IPADIC. + +## Step 1: download the archive + +Either fetch the archive with any tool you like, or use the helper next to this file, which adds checksum verification: + +``` +./download-mecab-dictionary.sh ipadic.tar.gz [expected-sha256] +``` + +On the first run without a checksum the script prints the SHA-256 it computed; record it and pass it on later runs so a changed or corrupted download fails loudly instead of being installed. + +## Step 2: unpack with the installer + +The installer extracts only the files a `MecabDictionary` reads (`*.csv`, `*.def`, and `dicrc`), flattens them into the target directory, and by the same flattening makes it impossible for an archive path to escape that directory: + +```java +import java.nio.file.Path; +import opennlp.tools.tokenize.lattice.MecabDictionaryInstaller; + +int files = MecabDictionaryInstaller.install( + Path.of("ipadic.tar.gz").toUri(), Path.of("ipadic")); +``` + +The returned count is the number of dictionary files extracted. A remote URI works in the same call if you prefer to skip step 1 entirely and let the installer stream the download. + +## Step 3: load and tokenize + +```java +import java.nio.charset.Charset; +import java.nio.file.Path; +import opennlp.tools.tokenize.lattice.LatticeTokenizer; +import opennlp.tools.tokenize.lattice.MecabDictionary; + +MecabDictionary dictionary = + MecabDictionary.load(Path.of("ipadic"), Charset.forName("EUC-JP")); +LatticeTokenizer tokenizer = new LatticeTokenizer(dictionary); +String[] tokens = tokenizer.tokenize("東京都に行く"); +``` + +For a UTF-8 dictionary such as mecab-ko-dic, `MecabDictionary.load(Path.of("ko-dic"))` is enough. Loaded dictionaries and tokenizers are immutable and safe to share between threads, so load once and reuse. + +## Chinese: the unigram segmenter needs only a frequency lexicon + +`opennlp.tools.tokenize.lattice.UnigramSegmenter` does not use mecab dictionaries. It loads a plain text lexicon, one entry per line: the word, its count, and optionally a tag, separated by whitespace. Any word-frequency list you have the rights to use works: + +```java +import java.nio.file.Path; +import opennlp.tools.tokenize.lattice.UnigramSegmenter; + +UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt")); +String[] tokens = segmenter.tokenize("我来到北京天安门"); +``` + +As with the dictionaries, the lexicon's license is between you and its publisher; nothing is bundled. diff --git a/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh b/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh new file mode 100755 index 0000000000..30c5b23bde --- /dev/null +++ b/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Downloads a mecab-format dictionary archive for the lattice tokenizer. +# +# This script only fetches the archive and, when a checksum is given, verifies it. +# It does not unpack anything: unpacking is the job of +# opennlp.tools.tokenize.lattice.MecabDictionaryInstaller, which extracts exactly the +# files a MecabDictionary reads and nothing else. See README-mecab-dictionaries.md in +# this directory for the known dictionary projects, their encodings, and the Java +# steps that follow the download. +# +# The dictionary you download carries its own license, which you accept by using it. +# Nothing is bundled with Apache OpenNLP and no dictionary location is built in. + +set -euo pipefail + +usage() { + echo "usage: $0 [expected-sha256]" >&2 + echo "" >&2 + echo " archive-url where to fetch the gzip-compressed tar from" >&2 + echo " target-file where to store the downloaded archive" >&2 + echo " expected-sha256 optional checksum; when given, the download is verified" >&2 + echo " and a mismatch deletes the file and fails the script" >&2 + exit 2 +} + +[ $# -lt 2 ] && usage +url="$1" +target="$2" +expected="${3:-}" + +# Download to a temporary name first so an interrupted transfer never leaves a +# half-written file at the target path. +tmp="${target}.download" +echo "downloading ${url}" +curl --fail --location --retry 3 --output "${tmp}" "${url}" + +actual="$(sha256sum "${tmp}" | cut -d' ' -f1)" +if [ -n "${expected}" ]; then + if [ "${actual}" != "${expected}" ]; then + rm -f "${tmp}" + echo "checksum mismatch: expected ${expected}" >&2 + echo " but got ${actual}" >&2 + exit 1 + fi + echo "checksum verified: ${actual}" +else + echo "sha256 of the download: ${actual}" + echo "record this value and pass it as the third argument next time to verify" +fi + +mv "${tmp}" "${target}" +echo "stored ${target}" +echo "" +echo "next: unpack and load it from Java; see README-mecab-dictionaries.md" From 7a4ee6e70ff2a2ec7573e225e5fd480576838974 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 22:09:30 -0400 Subject: [PATCH 07/14] OPENNLP-1894: Categorize by code point, keep unknown candidates inside their category run, and validate context ids at load --- .../tokenize/lattice/LatticeTokenizer.java | 47 +++- .../tokenize/lattice/MecabDictionary.java | 263 +++++++++++++++--- .../lattice/LatticeTokenizerTest.java | 181 ++++++++++++ 3 files changed, 442 insertions(+), 49 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index cb610f74fe..70d9c774f9 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -212,16 +212,20 @@ private List candidates(String text, int from, int to, int offset) { }); final boolean lexiconMatch = matched[0]; - final Category category = dictionary.categoryOf(text.charAt(position)); + final int codePoint = text.codePointAt(position); + final Category category = dictionary.categoryOf(codePoint); if (!lexiconMatch || category.invoke()) { - int run = position + 1; - while (run < to - && dictionary.categoryOf(text.charAt(run)).name().equals(category.name())) { - run++; + int run = position + Character.charCount(codePoint); + while (run < to) { + final int next = text.codePointAt(run); + if (!dictionary.categoryOf(next).name().equals(category.name())) { + break; + } + run += Character.charCount(next); } final List templates = dictionary.unknownEntries(category.name()); if (templates != null) { - addUnknown(candidates, position, run, to, category, templates); + addUnknown(candidates, text, position, run, category, templates); } } if (candidates.isEmpty()) { @@ -230,7 +234,8 @@ private List candidates(String text, int from, int to, int offset) { final List fallback = dictionary.unknownEntries("DEFAULT"); if (fallback != null) { for (final WordEntry entry : fallback) { - candidates.add(new Node(position, position + 1, entry, true)); + candidates.add( + new Node(position, position + Character.charCount(codePoint), entry, true)); } } } @@ -241,22 +246,38 @@ private List candidates(String text, int from, int to, int offset) { return candidates; } - /** Emits unknown-word candidates per the category's grouping and length settings. */ - private static void addUnknown(List candidates, int position, int runEnd, - int to, Category category, List templates) { + /** + * Emits unknown-word candidates per the category's grouping and length settings. + * + *

Every candidate stays inside the same-category run, so an unknown word never + * glues characters of different categories together, and every length counts whole + * characters rather than code units.

+ * + * @param candidates Receives the candidates. + * @param text The text being segmented. + * @param position The position the candidates start at. + * @param runEnd The exclusive end of the same-category run starting at + * {@code position}. + * @param category The category of that run. + * @param templates The category's unknown-word templates. + */ + private static void addUnknown(List candidates, String text, int position, + int runEnd, Category category, List templates) { if (category.group()) { for (final WordEntry entry : templates) { candidates.add(new Node(position, runEnd, entry, true)); } } final int lengths = category.length(); - for (int length = 1; length <= lengths && position + length <= to; length++) { - if (category.group() && position + length == runEnd) { + int end = position; + for (int length = 1; length <= lengths && end < runEnd; length++) { + end += Character.charCount(text.codePointAt(end)); + if (category.group() && end == runEnd) { // This length coincides with the grouped run emitted above; skip the duplicate. continue; } for (final WordEntry entry : templates) { - candidates.add(new Node(position, position + length, entry, true)); + candidates.add(new Node(position, end, entry, true)); } } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 908bc75f2c..c8138d7cf4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -24,6 +24,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -63,6 +64,146 @@ private static final class TrieNode { private List entries; } + /** + * The {@code char.def} code point to category name mapping, over the whole Unicode + * code point range. + * + *

The Basic Multilingual Plane is held in a directly indexed array, which is one + * reference per BMP code point and is the entire mapping for a BMP-only dictionary. + * The supplementary planes are held as a sorted, non-overlapping range table searched + * by binary search, because dictionaries map them in a handful of large blocks: a + * directly indexed array over all of Unicode would spend more than four megabytes of + * references to say what a few range rows already say.

+ */ + private static final class CategoryTable { + + private final String[] bmp; + private final int[] rangeStart; + private final int[] rangeEnd; + private final String[] rangeCategory; + + private CategoryTable(String[] bmp, int[] rangeStart, int[] rangeEnd, + String[] rangeCategory) { + this.bmp = bmp; + this.rangeStart = rangeStart; + this.rangeEnd = rangeEnd; + this.rangeCategory = rangeCategory; + } + + /** + * Looks up the category name a {@code char.def} mapping gives a code point. + * + * @param codePoint The code point to classify. + * @return The category name, or {@code null} when no mapping covers the code point. + */ + private String categoryOf(int codePoint) { + if (codePoint <= Character.MAX_VALUE) { + return bmp[codePoint]; + } + int low = 0; + int high = rangeStart.length - 1; + while (low <= high) { + final int middle = (low + high) >>> 1; + if (codePoint < rangeStart[middle]) { + high = middle - 1; + } else if (codePoint > rangeEnd[middle]) { + low = middle + 1; + } else { + return rangeCategory[middle]; + } + } + return null; + } + } + + /** + * Collects {@code char.def} mappings in file order and folds them into a + * {@link CategoryTable}, giving a later mapping precedence over an earlier one that + * covers the same code point, which is what direct indexing does for the BMP. + */ + private static final class CategoryTableBuilder { + + private final String[] bmp = new String[Character.MAX_VALUE + 1]; + private final List bounds = new ArrayList<>(); + private final List names = new ArrayList<>(); + + /** + * Records one inclusive code point range's category. + * + * @param from The first code point of the range. + * @param to The last code point of the range, inclusive. + * @param category The category name to give the range. Must not be {@code null}. + */ + private void map(int from, int to, String category) { + for (int c = from; c <= Math.min(to, Character.MAX_VALUE); c++) { + bmp[c] = category; + } + if (to > Character.MAX_VALUE) { + bounds.add(new int[] {Math.max(from, Character.MAX_VALUE + 1), to}); + names.add(category); + } + } + + /** + * Folds the recorded mappings into their lookup table. + * + * @return The table. Never {@code null}. + */ + private CategoryTable build() { + // Cut the supplementary ranges at every boundary they introduce, so that each + // resulting elementary interval is covered by a single winning range and the + // table stays sorted and non-overlapping for binary search. + final int[] edges = new int[bounds.size() * 2]; + for (int i = 0; i < bounds.size(); i++) { + edges[i * 2] = bounds.get(i)[0]; + edges[i * 2 + 1] = bounds.get(i)[1] + 1; + } + Arrays.sort(edges); + final List intervals = new ArrayList<>(); + final List categories = new ArrayList<>(); + for (int i = 0; i < edges.length - 1; i++) { + if (edges[i] == edges[i + 1]) { + continue; + } + final String winner = lastCovering(edges[i]); + if (winner == null) { + continue; + } + final int previous = intervals.size() - 1; + if (previous >= 0 && intervals.get(previous)[1] == edges[i] - 1 + && categories.get(previous).equals(winner)) { + intervals.get(previous)[1] = edges[i + 1] - 1; + } else { + intervals.add(new int[] {edges[i], edges[i + 1] - 1}); + categories.add(winner); + } + } + final int[] starts = new int[intervals.size()]; + final int[] ends = new int[intervals.size()]; + for (int i = 0; i < intervals.size(); i++) { + starts[i] = intervals.get(i)[0]; + ends[i] = intervals.get(i)[1]; + } + return new CategoryTable(bmp, starts, ends, categories.toArray(new String[0])); + } + + /** + * Finds the category of the last recorded range covering a code point. + * + * @param codePoint The code point to look up. + * @return The category name, or {@code null} when no recorded range covers it. + */ + private String lastCovering(int codePoint) { + for (int i = bounds.size() - 1; i >= 0; i--) { + final int[] range = bounds.get(i); + if (codePoint >= range[0] && codePoint <= range[1]) { + return names.get(i); + } + } + return null; + } + } + /** Receives one common-prefix match during {@link #prefixMatches}. */ interface PrefixMatchConsumer { @@ -80,18 +221,18 @@ interface PrefixMatchConsumer { private final short[] connectionCosts; private final int rightSize; private final Map categories; - private final String[] categoryOfChar; + private final CategoryTable categoryTable; private final Map> unknownEntries; private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, short[] connectionCosts, int rightSize, Map categories, - String[] categoryOfChar, Map> unknownEntries) { + CategoryTable categoryTable, Map> unknownEntries) { this.lexicon = lexicon; this.maxSurfaceLength = maxSurfaceLength; this.connectionCosts = connectionCosts; this.rightSize = rightSize; this.categories = categories; - this.categoryOfChar = categoryOfChar; + this.categoryTable = categoryTable; this.unknownEntries = unknownEntries; } @@ -116,25 +257,17 @@ public static MecabDictionary load(Path directory) throws IOException { * @param charset The encoding the distribution uses, for example UTF-8 or EUC-JP. * Must not be {@code null}. * @return The loaded dictionary. Never {@code null}. - * @throws IOException Thrown if reading fails, a required file is missing, or a file - * is malformed. + * @throws IOException Thrown if reading fails, a required file is missing, a file is + * malformed, or a lexicon entry's context ids are outside the + * {@code matrix.def} dimensions. * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static MecabDictionary load(Path directory, Charset charset) throws IOException { if (directory == null || charset == null) { throw new IllegalArgumentException("directory and charset must not be null"); } - final Map> lexicon = new HashMap<>(); - int maxSurface = 0; - try (DirectoryStream csvFiles = Files.newDirectoryStream(directory, "*.csv")) { - for (final Path csv : csvFiles) { - maxSurface = Math.max(maxSurface, readLexicon(csv, charset, lexicon)); - } - } - if (lexicon.isEmpty()) { - throw new IOException("no lexicon entries found under " + directory); - } - + // The connection matrix is read first because its dimensions are what every + // lexicon entry's context ids have to be inside of. final List matrixLines = readLines(directory.resolve("matrix.def"), charset); if (matrixLines.isEmpty()) { throw new IOException("empty matrix.def under " + directory); @@ -145,6 +278,10 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } final int leftSize = parseInt(header[0], "matrix.def", 1); final int rightSize = parseInt(header[1], "matrix.def", 1); + if (leftSize < 1 || rightSize < 1) { + throw new IOException("matrix.def dimensions must be positive, got " + + leftSize + " " + rightSize); + } final short[] costs = new short[leftSize * rightSize]; for (int i = 1; i < matrixLines.size(); i++) { final String line = matrixLines.get(i).trim(); @@ -157,19 +294,36 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } final int right = parseInt(fields[0], "matrix.def", i + 1); final int left = parseInt(fields[1], "matrix.def", i + 1); + if (right < 0 || right >= leftSize || left < 0 || left >= rightSize) { + throw new IOException("malformed matrix.def line " + (i + 1) + + ": context ids " + right + " " + left + + " are outside the declared dimensions " + leftSize + " " + rightSize); + } costs[right * rightSize + left] = (short) parseInt(fields[2], "matrix.def", i + 1); } + final Map> lexicon = new HashMap<>(); + int maxSurface = 0; + try (DirectoryStream csvFiles = Files.newDirectoryStream(directory, "*.csv")) { + for (final Path csv : csvFiles) { + maxSurface = Math.max(maxSurface, + readLexicon(csv, charset, lexicon, leftSize, rightSize)); + } + } + if (lexicon.isEmpty()) { + throw new IOException("no lexicon entries found under " + directory); + } + final Map categories = new HashMap<>(); - final String[] categoryOfChar = new String[Character.MAX_VALUE + 1]; + final CategoryTableBuilder categoryTable = new CategoryTableBuilder(); readCharacterDefinition(directory.resolve("char.def"), charset, categories, - categoryOfChar); + categoryTable); final Map> unknown = new HashMap<>(); - readLexicon(directory.resolve("unk.def"), charset, unknown); + readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); return new MecabDictionary(buildTrie(lexicon), maxSurface, costs, rightSize, - categories, categoryOfChar, unknown); + categories, categoryTable.build(), unknown); } /** Folds the surface-keyed lexicon into a character trie for prefix search. */ @@ -186,9 +340,24 @@ private static TrieNode buildTrie(Map> lexicon) { return root; } - /** Reads one lexicon-format CSV file; returns the longest surface seen. */ + /** + * Reads one lexicon-format CSV file, rejecting any entry whose context ids the + * connection matrix cannot be indexed with. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @param target Receives the entries, keyed by surface form. + * @param leftSize The first {@code matrix.def} dimension, which bounds right context + * ids. + * @param rightSize The second {@code matrix.def} dimension, which bounds left context + * ids. + * @return The length in characters of the longest surface form read. + * @throws IOException Thrown if the file is missing, an entry is malformed, or an + * entry's context id is outside the matrix dimensions. + */ private static int readLexicon(Path file, Charset charset, - Map> target) throws IOException { + Map> target, int leftSize, int rightSize) + throws IOException { int maxSurface = 0; int lineNumber = 0; for (final String line : readLines(file, charset)) { @@ -204,9 +373,19 @@ private static int readLexicon(Path file, Charset charset, if (surface.isEmpty()) { continue; } - final WordEntry entry = new WordEntry( - parseInt(fields.get(1), file.toString(), lineNumber), - parseInt(fields.get(2), file.toString(), lineNumber), + final int leftId = parseInt(fields.get(1), file.toString(), lineNumber); + final int rightId = parseInt(fields.get(2), file.toString(), lineNumber); + if (leftId < 0 || leftId >= rightSize) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": left context id " + leftId + " is outside the matrix.def dimensions " + + leftSize + " " + rightSize); + } + if (rightId < 0 || rightId >= leftSize) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": right context id " + rightId + " is outside the matrix.def dimensions " + + leftSize + " " + rightSize); + } + final WordEntry entry = new WordEntry(leftId, rightId, parseInt(fields.get(3), file.toString(), lineNumber), List.copyOf(fields.subList(4, fields.size()))); target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); @@ -217,7 +396,8 @@ private static int readLexicon(Path file, Charset charset, /** Reads char.def: category behavior lines and code point mapping lines. */ private static void readCharacterDefinition(Path file, Charset charset, - Map categories, String[] categoryOfChar) throws IOException { + Map categories, CategoryTableBuilder categoryTable) + throws IOException { int lineNumber = 0; for (final String raw : readLines(file, charset)) { lineNumber++; @@ -240,9 +420,11 @@ private static void readCharacterDefinition(Path file, Charset charset, if (fields.length < 2) { throw new IOException("mapping without category at " + file + " line " + lineNumber); } - for (int c = from; c <= to && c <= Character.MAX_VALUE; c++) { - categoryOfChar[c] = fields[1]; + if (from > to) { + throw new IOException("code point range descends at " + file + " line " + + lineNumber); } + categoryTable.map(from, to, fields[1]); } else { if (fields.length < 4) { throw new IOException("malformed category at " + file + " line " + lineNumber); @@ -310,13 +492,16 @@ int connectionCost(int rightId, int leftId) { } /** - * Classifies a character. + * Classifies a character by code point, so that a character outside the Basic + * Multilingual Plane is classified as the one character it is rather than as its two + * surrogates. * - * @param c The character. - * @return Its category, falling back to {@code DEFAULT}. Never {@code null}. + * @param codePoint The code point to classify. + * @return Its category, falling back to {@code DEFAULT} when no {@code char.def} + * mapping covers the code point. Never {@code null}. */ - Category categoryOf(char c) { - final String name = categoryOfChar[c]; + Category categoryOf(int codePoint) { + final String name = categoryTable.categoryOf(codePoint); final Category category = name == null ? null : categories.get(name); return category != null ? category : categories.get("DEFAULT"); } @@ -436,15 +621,21 @@ private static int parseInt(String text, String file, int lineNumber) * @param text The field text including the {@code 0x} prefix. * @param file The file being read, for the error message. * @param lineNumber The line being read, for the error message. - * @return The parsed code point. - * @throws IOException Thrown if the field is not a valid hexadecimal code point. + * @return The parsed code point, which may be in a supplementary plane. + * @throws IOException Thrown if the field is not a valid hexadecimal code point or + * names a value no Unicode code point has. */ private static int parseCodePoint(String text, Path file, int lineNumber) throws IOException { + final int codePoint; try { - return Integer.parseInt(text.trim().substring(2), 16); + codePoint = Integer.parseInt(text.trim().substring(2), 16); } catch (RuntimeException e) { throw new IOException("malformed code point in " + file + " line " + lineNumber, e); } + if (!Character.isValidCodePoint(codePoint)) { + throw new IOException("code point out of range in " + file + " line " + lineNumber); + } + return codePoint; } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 83b823706c..bf7a5f8c39 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -119,6 +119,37 @@ void testUnknownKanjiPreferOneMorphemeOverTwo() { Assertions.assertEquals(true, morphemes.get(0).unknown()); } + /** + * Verifies that an unknown-word candidate never spans a character category boundary. + * An unlisted kanji directly followed by a Latin letter must be analyzed as two + * morphemes of their own categories, never as one KANJI morpheme whose surface glues + * the kanji to the letter. + */ + @Test + void testUnknownCandidatesNeverSpanCategoryBoundaries() { + final String text = "\u5CE0a"; + Assertions.assertArrayEquals(new String[] {"\u5CE0", "a"}, tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1), new Span(1, 2)}, + tokenizer.tokenizePos(text)); + final List morphemes = tokenizer.analyze(text); + Assertions.assertEquals(List.of("noun", "unknown"), morphemes.get(0).features()); + Assertions.assertEquals(List.of("noun", "foreign"), morphemes.get(1).features()); + } + + /** + * Verifies that bounding unknown-word candidates by the category run does not under + * generate inside the run: a two-kanji unlisted run followed by a Latin letter still + * offers the length-two KANJI candidate, which wins over two single-kanji morphemes. + */ + @Test + void testUnknownRunStillOffersWithinCategoryLengths() { + final String text = "\u5CE0\u9053a"; + Assertions.assertArrayEquals(new String[] {"\u5CE0\u9053", "a"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 2), new Span(2, 3)}, + tokenizer.tokenizePos(text)); + } + @Test void testWhitespaceSeparatesAndIsNeverAMorpheme() { final String text = "\u6771\u4EAC \u306B \u884C\u304F"; @@ -216,6 +247,8 @@ void testSpansStayOriginalAfterLeadingWhitespace() { */ @Test void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { + // The rest of the dictionary is well formed, so the short row is what load rejects. + writeUnitMatrixDictionary(broken); Files.write(broken.resolve("lexicon.csv"), "\u6771,0,0\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); @@ -227,6 +260,8 @@ void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { */ @Test void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { + // The rest of the dictionary is well formed, so the cost column is what load rejects. + writeUnitMatrixDictionary(broken); Files.write(broken.resolve("lexicon.csv"), "\u6771,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); @@ -295,6 +330,152 @@ void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } + /** + * Writes a miniature dictionary whose {@code char.def} maps a supplementary plane + * range, the shape a UniDic-style distribution uses for the CJK extension blocks. + * + * @param target The directory to write the dictionary files into. Must not be + * {@code null} and must exist. + * @throws IOException Thrown if writing any of the files fails. + */ + private static void writeSupplementaryDictionary(Path target) throws IOException { + Files.write(target.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun,common\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "LATIN 1 1 0", + "", + "0x4E00..0x9FFF KANJI", + "0x20000..0x2A6DF KANJI", + "0x0061..0x007A LATIN", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("unk.def"), String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "LATIN,0,0,4000,noun,foreign", + "").getBytes(StandardCharsets.UTF_8)); + } + + /** + * Verifies that a {@code char.def} range above U+FFFF is honored rather than + * discarded: a supplementary plane ideograph inside the mapped range takes the + * category the range names, while a supplementary code point outside every mapped + * range still falls back to DEFAULT. + */ + @Test + void testSupplementaryCharDefRangeIsHonored(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final MecabDictionary dictionary = MecabDictionary.load(supplementary); + // U+20BB7 is a CJK extension B ideograph inside the mapped range. + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x20BB7).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x6771).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x2460).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x2A6E0).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf('a').name()); + } + + /** + * Verifies that a supplementary plane ideograph is analyzed as the single character + * it is: one morpheme whose span covers both code units and which carries the + * features of the category its {@code char.def} range names, never one morpheme per + * surrogate. The second case shows the category's length templates count characters, + * not code units, so a run of two supplementary ideographs is still reachable by the + * length-two template. + */ + @Test + void testSupplementaryIdeographIsOneMorpheme(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final LatticeTokenizer supplementaryTokenizer = + new LatticeTokenizer(MecabDictionary.load(supplementary)); + // U+20BB7 written as its surrogate pair, per this file's ASCII-only convention. + final String text = "\uD842\uDFB7"; + final List morphemes = supplementaryTokenizer.analyze(text); + Assertions.assertEquals(1, morphemes.size()); + Assertions.assertEquals(text, morphemes.get(0).surface()); + Assertions.assertEquals(new Span(0, 2), morphemes.get(0).span()); + Assertions.assertEquals(List.of("noun", "unknown"), morphemes.get(0).features()); + + final List pair = supplementaryTokenizer.analyze(text + text); + Assertions.assertEquals(1, pair.size()); + Assertions.assertEquals(new Span(0, 4), pair.get(0).span()); + Assertions.assertEquals(List.of("noun", "unknown"), pair.get(0).features()); + } + + /** + * Verifies that a supplementary plane ideograph does not absorb neighbouring text of + * another category: the ideograph and an unmapped symbol beside it stay two + * morphemes, each span covering whole characters. + */ + @Test + void testSupplementaryIdeographDoesNotAbsorbItsNeighbour(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final LatticeTokenizer supplementaryTokenizer = + new LatticeTokenizer(MecabDictionary.load(supplementary)); + final String text = "\uD842\uDFB7\u2460"; + Assertions.assertArrayEquals(new String[] {"\uD842\uDFB7", "\u2460"}, + supplementaryTokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 2), new Span(2, 3)}, + supplementaryTokenizer.tokenizePos(text)); + } + + /** + * Writes every dictionary file except the lexicon, so a test can supply a lexicon of + * its own against a one by one connection matrix. + * + * @param target The directory to write the dictionary files into. Must not be + * {@code null} and must exist. + * @throws IOException Thrown if writing any of the files fails. + */ + private static void writeUnitMatrixDictionary(Path target) throws IOException { + Files.write(target.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("char.def"), + "DEFAULT 0 1 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + } + + /** + * Verifies that a lexicon row whose right context id is outside the + * {@code matrix.def} dimensions is rejected at load time, naming the file, the line, + * and the offending id, rather than reaching the cost matrix with an out of range + * index during segmentation. + */ + @Test + void testRightContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) + throws IOException { + writeUnitMatrixDictionary(mismatched); + Files.write(mismatched.resolve("lexicon.csv"), + "\u6771,0,5,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(mismatched)); + Assertions.assertEquals("malformed entry at " + mismatched.resolve("lexicon.csv") + + " line 1: right context id 5 is outside the matrix.def dimensions 1 1", + e.getMessage()); + } + + /** + * Verifies that a lexicon row whose left context id is outside the {@code matrix.def} + * dimensions is rejected at load time, naming the file, the line, and the offending + * id. + */ + @Test + void testLeftContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) + throws IOException { + writeUnitMatrixDictionary(mismatched); + Files.write(mismatched.resolve("lexicon.csv"), + "\u6771,0,0,3000,noun\n\u90FD,7,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(mismatched)); + Assertions.assertEquals("malformed entry at " + mismatched.resolve("lexicon.csv") + + " line 2: left context id 7 is outside the matrix.def dimensions 1 1", + e.getMessage()); + } + @Test void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class, From d8870784b8f291a41bfb3f40169ce1d673e9d557 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 00:53:57 -0400 Subject: [PATCH 08/14] OPENNLP-1894: Precompute category runs, unbox the trie, and reject inexpressible dictionary values The lattice tokenizer rescanned the same-category run from every position, so a run of L characters cost on the order of L squared category lookups; a 16,000-character katakana run measured around half a second. One right-to-left pass per stretch now fixes every position's category and run end, and the same 16,000-character run tokenizes in about half a millisecond at 31 million characters per second. The character table holds Category instances instead of names, so the per-character path compares by identity with no name-map lookup, and a char.def mapping to an undefined category now fails at load naming the code point. The lexicon trie's children are sorted character arrays found by binary search, so a descent no longer boxes a Character per step. matrix.def loading rejects connection costs outside the 16-bit range instead of silently truncating them, and dimension products beyond the addressable array size fail at the header. The unigram segmenter's unknown-character fallback advances one code point, never one code unit, so an unknown supplementary character is stepped over whole and no span can split its surrogate halves. --- .../tokenize/lattice/LatticeTokenizer.java | 66 ++++++-- .../tokenize/lattice/MecabDictionary.java | 141 ++++++++++++++---- .../tokenize/lattice/UnigramSegmenter.java | 11 +- .../lattice/LatticeTokenizerTest.java | 138 +++++++++++++++++ .../lattice/UnigramSegmenterTest.java | 24 +++ 5 files changed, 337 insertions(+), 43 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index 70d9c774f9..f7a479bf83 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -143,11 +143,19 @@ private void decode(String text, int from, int to, List morphemes) { final boolean[] reachable = new boolean[length + 1]; reachable[0] = true; + // One right-to-left pass fixes each position's category and same-category run end, + // so candidate generation reads them instead of rescanning the run from every + // position, which would cost the square of the run length on long uniform runs. + final Category[] categoryAt = new Category[length]; + final int[] runEndAt = new int[length]; + computeCategoryRuns(text, from, to, categoryAt, runEndAt); + for (int i = 0; i < length; i++) { if (!reachable[i]) { continue; } - final List candidates = candidates(text, from, to, i); + final List candidates = + candidates(text, from, to, i, categoryAt[i], runEndAt[i]); for (final Node candidate : candidates) { relax(candidate, i == 0 ? null : endingAt.get(i)); if (candidate.pathCost < Long.MAX_VALUE) { @@ -199,8 +207,39 @@ private void relax(Node candidate, List predecessors) { } } + /** + * Fills the per-position category and same-category run end for one stretch, in one + * right-to-left pass over its code points. Positions inside a surrogate pair keep a + * {@code null} category; no candidate ever starts there. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param categoryAt Receives each position's category, indexed by {@code + * position - from}. + * @param runEndAt Receives each position's exclusive same-category run end, indexed + * the same way. + */ + private void computeCategoryRuns(String text, int from, int to, + Category[] categoryAt, int[] runEndAt) { + int next = -1; + for (int position = to; position > from; ) { + final int codePoint = text.codePointBefore(position); + position -= Character.charCount(codePoint); + final int index = position - from; + categoryAt[index] = dictionary.categoryOf(codePoint); + if (next >= 0 && categoryAt[next] == categoryAt[index]) { + runEndAt[index] = runEndAt[next]; + } else { + runEndAt[index] = next >= 0 ? next + from : to; + } + next = index; + } + } + /** Gathers lexicon matches and unknown-word candidates starting at one position. */ - private List candidates(String text, int from, int to, int offset) { + private List candidates(String text, int from, int to, int offset, + Category positionCategory, int positionRunEnd) { final int position = from + offset; final List candidates = new ArrayList<>(); final boolean[] matched = new boolean[1]; @@ -213,19 +252,22 @@ private List candidates(String text, int from, int to, int offset) { final boolean lexiconMatch = matched[0]; final int codePoint = text.codePointAt(position); - final Category category = dictionary.categoryOf(codePoint); + final Category category; + final int runEnd; + if (positionCategory == null) { + // Only a lexicon surface ending inside a surrogate pair could make such a + // position reachable; classify the stray code unit on the spot so the lattice + // stays connected the way it always did. + category = dictionary.categoryOf(codePoint); + runEnd = position + Character.charCount(codePoint); + } else { + category = positionCategory; + runEnd = positionRunEnd; + } if (!lexiconMatch || category.invoke()) { - int run = position + Character.charCount(codePoint); - while (run < to) { - final int next = text.codePointAt(run); - if (!dictionary.categoryOf(next).name().equals(category.name())) { - break; - } - run += Character.charCount(next); - } final List templates = dictionary.unknownEntries(category.name()); if (templates != null) { - addUnknown(candidates, text, position, run, category, templates); + addUnknown(candidates, text, position, runEnd, category, templates); } } if (candidates.isEmpty()) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index c8138d7cf4..4a34e24d27 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -58,10 +58,56 @@ record WordEntry(int leftId, int rightId, int cost, List features) { record Category(String name, boolean invoke, boolean group, int length) { } - /** One node of the lexicon trie, keyed by the next surface character. */ + /** + * One frozen node of the lexicon trie: children are held as a sorted character + * array with a parallel node array and found by binary search, so a descent never + * boxes the character the way a map keyed by {@link Character} would on every step + * of the innermost matching loop. + */ private static final class TrieNode { - private final Map children = new HashMap<>(); + + private final char[] keys; + private final TrieNode[] nodes; + private final List entries; + + private TrieNode(char[] keys, TrieNode[] nodes, List entries) { + this.keys = keys; + this.nodes = nodes; + this.entries = entries; + } + + /** + * Descends one character. + * + * @param c The next surface character. + * @return The child node, or {@code null} when no surface continues with {@code c}. + */ + private TrieNode child(char c) { + final int index = Arrays.binarySearch(keys, c); + return index >= 0 ? nodes[index] : null; + } + } + + /** One mutable trie node during construction, frozen into a {@link TrieNode}. */ + private static final class TrieBuilderNode { + + private final Map children = new HashMap<>(); private List entries; + + /** Freezes this node and its subtree into the sorted-array form. */ + private TrieNode freeze() { + final char[] keys = new char[children.size()]; + int i = 0; + for (final Character key : children.keySet()) { + keys[i++] = key; + } + Arrays.sort(keys); + final TrieNode[] nodes = new TrieNode[keys.length]; + for (int k = 0; k < keys.length; k++) { + nodes[k] = children.get(keys[k]).freeze(); + } + return new TrieNode(keys, nodes, entries); + } } /** @@ -77,13 +123,13 @@ private static final class TrieNode { */ private static final class CategoryTable { - private final String[] bmp; + private final Category[] bmp; private final int[] rangeStart; private final int[] rangeEnd; - private final String[] rangeCategory; + private final Category[] rangeCategory; - private CategoryTable(String[] bmp, int[] rangeStart, int[] rangeEnd, - String[] rangeCategory) { + private CategoryTable(Category[] bmp, int[] rangeStart, int[] rangeEnd, + Category[] rangeCategory) { this.bmp = bmp; this.rangeStart = rangeStart; this.rangeEnd = rangeEnd; @@ -91,12 +137,16 @@ private CategoryTable(String[] bmp, int[] rangeStart, int[] rangeEnd, } /** - * Looks up the category name a {@code char.def} mapping gives a code point. + * Looks up the category a {@code char.def} mapping gives a code point. The table + * holds the {@link Category} instances themselves, so a lookup on the tokenizer's + * per-character path costs one array read or one binary search, never a name map + * access, and two code points of one category share one instance to compare by + * identity. * * @param codePoint The code point to classify. - * @return The category name, or {@code null} when no mapping covers the code point. + * @return The category, or {@code null} when no mapping covers the code point. */ - private String categoryOf(int codePoint) { + private Category categoryOf(int codePoint) { if (codePoint <= Character.MAX_VALUE) { return bmp[codePoint]; } @@ -149,7 +199,7 @@ private void map(int from, int to, String category) { * * @return The table. Never {@code null}. */ - private CategoryTable build() { + private CategoryTable build(Map categories) throws IOException { // Cut the supplementary ranges at every boundary they introduce, so that each // resulting elementary interval is covered by a single winning range and the // table stays sorted and non-overlapping for binary search. @@ -160,7 +210,7 @@ private CategoryTable build() { } Arrays.sort(edges); final List intervals = new ArrayList<>(); - final List categories = new ArrayList<>(); + final List winners = new ArrayList<>(); for (int i = 0; i < edges.length - 1; i++) { if (edges[i] == edges[i + 1]) { continue; @@ -171,11 +221,11 @@ private CategoryTable build() { } final int previous = intervals.size() - 1; if (previous >= 0 && intervals.get(previous)[1] == edges[i] - 1 - && categories.get(previous).equals(winner)) { + && winners.get(previous).equals(winner)) { intervals.get(previous)[1] = edges[i + 1] - 1; } else { intervals.add(new int[] {edges[i], edges[i + 1] - 1}); - categories.add(winner); + winners.add(winner); } } final int[] starts = new int[intervals.size()]; @@ -184,7 +234,32 @@ private CategoryTable build() { starts[i] = intervals.get(i)[0]; ends[i] = intervals.get(i)[1]; } - return new CategoryTable(bmp, starts, ends, categories.toArray(new String[0])); + final Category[] resolvedBmp = new Category[bmp.length]; + for (int c = 0; c < bmp.length; c++) { + if (bmp[c] != null) { + resolvedBmp[c] = resolve(bmp[c], categories, c); + } + } + final Category[] resolvedRanges = new Category[winners.size()]; + for (int i = 0; i < winners.size(); i++) { + resolvedRanges[i] = resolve(winners.get(i), categories, starts[i]); + } + return new CategoryTable(resolvedBmp, starts, ends, resolvedRanges); + } + + /** + * Resolves a mapped category name against the defined categories, so a mapping to + * a name the {@code char.def} category section never defined fails at load with + * the offending code point instead of falling back silently at lookup time. + */ + private static Category resolve(String name, Map categories, + int codePoint) throws IOException { + final Category category = categories.get(name); + if (category == null) { + throw new IOException(String.format( + "char.def maps U+%04X to the undefined category %s", codePoint, name)); + } + return category; } /** @@ -222,6 +297,7 @@ interface PrefixMatchConsumer { private final int rightSize; private final Map categories; private final CategoryTable categoryTable; + private final Category defaultCategory; private final Map> unknownEntries; private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, @@ -233,6 +309,7 @@ private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, this.rightSize = rightSize; this.categories = categories; this.categoryTable = categoryTable; + this.defaultCategory = categories.get("DEFAULT"); this.unknownEntries = unknownEntries; } @@ -282,7 +359,12 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc throw new IOException("matrix.def dimensions must be positive, got " + leftSize + " " + rightSize); } - final short[] costs = new short[leftSize * rightSize]; + final long cells = (long) leftSize * rightSize; + if (cells > Integer.MAX_VALUE) { + throw new IOException("matrix.def dimensions " + leftSize + " x " + rightSize + + " overflow the addressable connection matrix"); + } + final short[] costs = new short[(int) cells]; for (int i = 1; i < matrixLines.size(); i++) { final String line = matrixLines.get(i).trim(); if (line.isEmpty()) { @@ -299,7 +381,13 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc + ": context ids " + right + " " + left + " are outside the declared dimensions " + leftSize + " " + rightSize); } - costs[right * rightSize + left] = (short) parseInt(fields[2], "matrix.def", i + 1); + final int cost = parseInt(fields[2], "matrix.def", i + 1); + if (cost < Short.MIN_VALUE || cost > Short.MAX_VALUE) { + throw new IOException("malformed matrix.def line " + (i + 1) + + ": connection cost " + cost + " is outside the 16-bit range the" + + " format defines"); + } + costs[right * rightSize + left] = (short) cost; } final Map> lexicon = new HashMap<>(); @@ -318,26 +406,26 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc final CategoryTableBuilder categoryTable = new CategoryTableBuilder(); readCharacterDefinition(directory.resolve("char.def"), charset, categories, categoryTable); - final Map> unknown = new HashMap<>(); readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); return new MecabDictionary(buildTrie(lexicon), maxSurface, costs, rightSize, - categories, categoryTable.build(), unknown); + categories, categoryTable.build(categories), unknown); } /** Folds the surface-keyed lexicon into a character trie for prefix search. */ private static TrieNode buildTrie(Map> lexicon) { - final TrieNode root = new TrieNode(); + final TrieBuilderNode root = new TrieBuilderNode(); for (final Map.Entry> entry : lexicon.entrySet()) { - TrieNode node = root; + TrieBuilderNode node = root; final String surface = entry.getKey(); for (int i = 0; i < surface.length(); i++) { - node = node.children.computeIfAbsent(surface.charAt(i), key -> new TrieNode()); + node = node.children.computeIfAbsent(surface.charAt(i), + key -> new TrieBuilderNode()); } node.entries = List.copyOf(entry.getValue()); } - return root; + return root.freeze(); } /** @@ -448,7 +536,7 @@ private static void readCharacterDefinition(Path file, Charset charset, List lookup(String surface) { TrieNode node = lexicon; for (int i = 0; i < surface.length() && node != null; i++) { - node = node.children.get(surface.charAt(i)); + node = node.child(surface.charAt(i)); } return node == null ? null : node.entries; } @@ -465,7 +553,7 @@ List lookup(String surface) { void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) { TrieNode node = lexicon; for (int i = from; i < to; i++) { - node = node.children.get(text.charAt(i)); + node = node.child(text.charAt(i)); if (node == null) { return; } @@ -501,9 +589,8 @@ int connectionCost(int rightId, int leftId) { * mapping covers the code point. Never {@code null}. */ Category categoryOf(int codePoint) { - final String name = categoryTable.categoryOf(codePoint); - final Category category = name == null ? null : categories.get(name); - return category != null ? category : categories.get("DEFAULT"); + final Category category = categoryTable.categoryOf(codePoint); + return category != null ? category : defaultCategory; } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index ea3d188bc5..b4cfaa8ea6 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -218,11 +218,14 @@ private void decode(String text, int from, int to, List spans) { continue; } // A single-character step at the unknown log-probability keeps every position - // reachable even where no lexicon word matches. + // reachable even where no lexicon word matches. The step advances one code + // point, never one code unit, so an unknown supplementary character is stepped + // over whole and no span boundary can land between its surrogate halves. + final int width = Character.charCount(text.codePointAt(from + i)); final double fallback = best[i] + unknownLogProbability; - if (fallback > best[i + 1]) { - best[i + 1] = fallback; - previous[i + 1] = i; + if (i + width <= length && fallback > best[i + width]) { + best[i + width] = fallback; + previous[i + width] = i; } WordTrie node = trie; for (int j = from + i; j < to; j++) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index bf7a5f8c39..e1be5a4a98 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -484,4 +484,142 @@ void testInvalidArguments() { () -> MecabDictionary.load(null)); Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.analyze(null)); } + + /** + * Verifies the supplementary range table's interval cutting and precedence: a later + * {@code char.def} mapping strictly inside an earlier one wins exactly on its own + * stretch, and the earlier category resumes after it, so the cut produces three + * intervals from two overlapping ranges. + */ + @Test + void testLaterSupplementaryMappingWinsInsideAnEarlierRange(@TempDir Path overlapped) + throws IOException { + Files.write(overlapped.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(overlapped.resolve("matrix.def"), + "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(overlapped.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "LATIN 1 1 0", + "", + "0x20000..0x2FFFF KANJI", + "0x24000..0x25000 LATIN", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(overlapped.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + + final MecabDictionary dictionary = MecabDictionary.load(overlapped); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x20000).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x23FFF).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x24000).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x25000).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x25001).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x2FFFF).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x30000).name()); + } + + /** + * Verifies a {@code char.def} range straddling the BMP boundary: the part up to + * U+FFFF lands in the directly indexed table and the rest in the range table, and + * both halves answer the same category with no gap at the seam. + */ + @Test + void testCharDefRangeStraddlingTheBmpBoundary(@TempDir Path straddling) + throws IOException { + Files.write(straddling.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(straddling.resolve("matrix.def"), + "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(straddling.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "LATIN 1 1 0", + "", + "0xFF00..0x10040 LATIN", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(straddling.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + + final MecabDictionary dictionary = MecabDictionary.load(straddling); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0xFF00).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0xFFFF).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x10000).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x10040).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x10041).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0xFEFF).name()); + } + + /** + * Verifies that a {@code char.def} mapping to a category its category section never + * defined fails at load, naming the code point and the ghost category, instead of + * silently falling back to DEFAULT at lookup time. + */ + @Test + void testMappingToUndefinedCategoryFailsLoud(@TempDir Path ghost) throws IOException { + Files.write(ghost.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(ghost.resolve("matrix.def"), + "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(ghost.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "", + "0x0100..0x0110 GHOST", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(ghost.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(ghost)); + Assertions.assertEquals("char.def maps U+0100 to the undefined category GHOST", + e.getMessage()); + } + + /** + * Verifies that a connection cost outside the 16-bit range the binary matrix format + * defines is rejected at load instead of being truncated by the narrowing cast into + * a silently different cost. + */ + @Test + void testMatrixCostOutsideShortRangeFailsLoud(@TempDir Path broken) throws IOException { + writeUnitMatrixDictionary(broken); + Files.write(broken.resolve("matrix.def"), + "1 1\n0 0 40000\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("malformed matrix.def line 2: connection cost 40000 is" + + " outside the 16-bit range the format defines", e.getMessage()); + } + + /** + * Verifies that {@code matrix.def} dimensions whose product exceeds the addressable + * array size fail loud at the header instead of overflowing the int multiplication + * into a negative or wrapped allocation size. + */ + @Test + void testMatrixDimensionProductBeyondIntRangeFailsLoud(@TempDir Path broken) + throws IOException { + writeUnitMatrixDictionary(broken); + Files.write(broken.resolve("matrix.def"), + "70000 70000\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("matrix.def dimensions 70000 x 70000 overflow the" + + " addressable connection matrix", e.getMessage()); + } + + /** + * Verifies that a {@code matrix.def} data row naming context ids outside the + * declared dimensions is rejected at load with the offending line and ids. + */ + @Test + void testMatrixRowContextIdsOutsideDimensionsFailLoud(@TempDir Path broken) + throws IOException { + writeUnitMatrixDictionary(broken); + Files.write(broken.resolve("matrix.def"), + "1 1\n2 0 5\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("malformed matrix.def line 2: context ids 2 0 are outside" + + " the declared dimensions 1 1", e.getMessage()); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index 9725b8c729..e6ebd604e1 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -176,4 +176,28 @@ void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class, () -> segmenter.tokenizePos(null)); } + + /** + * Verifies that the unknown-character fallback advances one code point, never one + * code unit: a supplementary character absent from the lexicon comes back as one + * span over both of its surrogate halves, and no span boundary ever lands between + * them. + */ + @org.junit.jupiter.api.Test + void testUnknownSupplementaryCharacterIsNeverSplit() { + // U+20BB7, a CJK extension B ideograph, written as its surrogate pair + final String text = "\uD842\uDFB7\uD842\uDFB7"; + final opennlp.tools.util.Span[] spans = segmenter.tokenizePos(text); + for (final opennlp.tools.util.Span span : spans) { + Assertions.assertEquals(0, span.getStart() % 2, + "span must start on a code point boundary: " + span); + Assertions.assertEquals(0, span.getEnd() % 2, + "span must end on a code point boundary: " + span); + } + int covered = 0; + for (final opennlp.tools.util.Span span : spans) { + covered += span.length(); + } + Assertions.assertEquals(text.length(), covered); + } } From 075b343c824965d53f3bffd7eced6d8fe0d866d1 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 06:54:45 -0400 Subject: [PATCH 09/14] OPENNLP-1894: Hold the lexicon in a double-array trie with frequency-recoded labels The lexicon trie's per-node child lookup, a binary search over the node's fan-out, paid about a dozen comparisons at the root of a real dictionary; the classic base/check double-array makes every transition one array read and one comparison. Characters are recoded into dense labels ordered by descending frequency before the array is built, so the array stays compact although CJK surfaces draw on tens of thousands of distinct characters, and a character the lexicon never uses misses in the recode table before the array is consulted. On the IPADIC harness the prefix walk now matches the fastest previous implementation at 5.6M chars/s with strictly constant-time transitions, and building the array adds about a quarter second to the 392k-entry load. --- .../tokenize/lattice/MecabDictionary.java | 304 ++++++++++++++---- 1 file changed, 236 insertions(+), 68 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 4a34e24d27..cd1f46ba7f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -59,54 +59,250 @@ record Category(String name, boolean invoke, boolean group, int length) { } /** - * One frozen node of the lexicon trie: children are held as a sorted character - * array with a parallel node array and found by binary search, so a descent never - * boxes the character the way a map keyed by {@link Character} would on every step - * of the innermost matching loop. + * The lexicon as a double-array trie: one transition is one array read and one + * comparison, so the per-position prefix walk of the tokenizer never hashes, never + * binary-searches a fan-out, and never boxes. Characters are recoded into dense + * labels ordered by frequency before the array is built, which keeps the array + * compact although CJK surfaces draw from tens of thousands of distinct + * characters; a character the lexicon never uses misses in the recode table before + * the array is even consulted. + * + *

The layout is the classic base/check pair: from state {@code s}, label + * {@code c} leads to {@code t = base[s] + c} exactly when {@code check[t] == s}. + * Label {@code 0} terminates a surface and leads to a state whose negative base + * encodes the index of the surface's entry list.

*/ - private static final class TrieNode { + private static final class DoubleArrayLexicon { + + private final int[] base; + private final int[] check; + private final int[] codeOf; + private final List[] values; + + private DoubleArrayLexicon(int[] base, int[] check, int[] codeOf, + List[] values) { + this.base = base; + this.check = check; + this.codeOf = codeOf; + this.values = values; + } + + /** + * Builds the trie from the surface-keyed lexicon. + * + * @param lexicon The entries keyed by surface form. + * @return The built trie. Never {@code null}. + */ + @SuppressWarnings("unchecked") + private static DoubleArrayLexicon build(Map> lexicon) { + final String[] surfaces = lexicon.keySet().toArray(new String[0]); + Arrays.sort(surfaces); + final List[] values = new List[surfaces.length]; + for (int i = 0; i < surfaces.length; i++) { + values[i] = List.copyOf(lexicon.get(surfaces[i])); + } - private final char[] keys; - private final TrieNode[] nodes; - private final List entries; + // Dense recode: labels ordered by descending frequency get the small codes, so + // busy transitions cluster at the low end of the array. + final int[] frequency = new int[Character.MAX_VALUE + 1]; + for (final String surface : surfaces) { + for (int i = 0; i < surface.length(); i++) { + frequency[surface.charAt(i)]++; + } + } + final Integer[] chars = new Integer[Character.MAX_VALUE + 1]; + int distinct = 0; + for (int c = 0; c <= Character.MAX_VALUE; c++) { + if (frequency[c] > 0) { + chars[distinct++] = c; + } + } + final Integer[] ordered = Arrays.copyOf(chars, distinct); + Arrays.sort(ordered, (a, b) -> frequency[b] - frequency[a]); + final int[] codeOf = new int[Character.MAX_VALUE + 1]; + Arrays.fill(codeOf, -1); + for (int rank = 0; rank < ordered.length; rank++) { + codeOf[ordered[rank]] = rank + 1; + } - private TrieNode(char[] keys, TrieNode[] nodes, List entries) { - this.keys = keys; - this.nodes = nodes; - this.entries = entries; + final Builder builder = new Builder(surfaces, codeOf); + builder.insert(0, surfaces.length, 0, Builder.ROOT); + return new DoubleArrayLexicon(Arrays.copyOf(builder.base, builder.high + 1), + Arrays.copyOf(builder.check, builder.high + 1), codeOf, values); } /** - * Descends one character. + * Reports every surface starting at a text position, walking the array once. * - * @param c The next surface character. - * @return The child node, or {@code null} when no surface continues with {@code c}. + * @param text The text being segmented. + * @param from The position surfaces must start at. + * @param to The exclusive end of the searchable stretch. + * @param consumer Receives each match length with its entries. */ - private TrieNode child(char c) { - final int index = Arrays.binarySearch(keys, c); - return index >= 0 ? nodes[index] : null; + private void prefixMatches(String text, int from, int to, + PrefixMatchConsumer consumer) { + int state = Builder.ROOT; + for (int i = from; i < to; i++) { + final char c = text.charAt(i); + final int code = codeOf[c]; + if (code < 0) { + return; + } + final int next = base[state] + code; + if (next >= check.length || check[next] != state) { + return; + } + state = next; + final int terminal = base[state]; + if (terminal < check.length && check[terminal] == state && base[terminal] < 0) { + consumer.accept(i - from + 1, values[-base[terminal] - 1]); + } + } + } + + /** + * Looks up the entries of an exact surface form. + * + * @param surface The surface form. + * @return The entries, or {@code null} when the surface is not listed. + */ + private List lookup(String surface) { + int state = Builder.ROOT; + for (int i = 0; i < surface.length(); i++) { + final int code = codeOf[surface.charAt(i)]; + if (code < 0) { + return null; + } + final int next = base[state] + code; + if (next >= check.length || check[next] != state) { + return null; + } + state = next; + } + final int terminal = base[state]; + if (terminal < check.length && check[terminal] == state && base[terminal] < 0) { + return values[-base[terminal] - 1]; + } + return null; } - } - /** One mutable trie node during construction, frozen into a {@link TrieNode}. */ - private static final class TrieBuilderNode { + /** + * The recursive sorted-range builder: each call places one node's children by + * finding a base at which every child label lands on a free slot, then recurses + * per child range. A moving watermark keeps the free-slot search near-linear over + * real lexicons. + */ + private static final class Builder { + + private static final int ROOT = 1; + private static final int EMPTY = -1; + + private final String[] surfaces; + private final int[] codeOf; + private int[] base; + private int[] check; + private int high = ROOT; + private int watermark = ROOT + 1; + private int valueIndex; + + private Builder(String[] surfaces, int[] codeOf) { + this.surfaces = surfaces; + this.codeOf = codeOf; + base = new int[1 << 16]; + check = new int[1 << 16]; + Arrays.fill(check, EMPTY); + } - private final Map children = new HashMap<>(); - private List entries; + /** + * Places the children of one trie node. + * + * @param left The first surface of the node's range. + * @param right The exclusive last surface of the node's range. + * @param depth The character depth of the node. + * @param state The node's own slot. + */ + private void insert(int left, int right, int depth, int state) { + // gather the distinct child labels of this range, terminator first + final int[] labels = new int[right - left]; + int labelCount = 0; + int previous = -2; + for (int k = left; k < right; k++) { + final int label = surfaces[k].length() == depth + ? 0 : codeOf[surfaces[k].charAt(depth)]; + if (label != previous) { + labels[labelCount++] = label; + previous = label; + } + } + final int found = findBase(labels, labelCount); + base[state] = found; + for (int k = 0; k < labelCount; k++) { + final int child = found + labels[k]; + check[child] = state; + if (child > high) { + high = child; + } + } + // recurse over each child's sub-range + int start = left; + for (int k = 0; k < labelCount; k++) { + final int label = labels[k]; + int end = start; + while (end < right && (surfaces[end].length() == depth + ? 0 : codeOf[surfaces[end].charAt(depth)]) == label) { + end++; + } + final int child = found + label; + if (label == 0) { + base[child] = -(++valueIndex); + } else { + insert(start, end, depth + 1, child); + } + start = end; + } + } - /** Freezes this node and its subtree into the sorted-array form. */ - private TrieNode freeze() { - final char[] keys = new char[children.size()]; - int i = 0; - for (final Character key : children.keySet()) { - keys[i++] = key; + /** + * Finds the lowest base at which every label lands on a free slot. Labels + * arrive in surface-character order, not numeric order, so the smallest and + * largest label are computed rather than assumed positional. + */ + private int findBase(int[] labels, int labelCount) { + int smallest = labels[0]; + int largest = labels[0]; + for (int k = 1; k < labelCount; k++) { + smallest = Math.min(smallest, labels[k]); + largest = Math.max(largest, labels[k]); + } + int candidate = Math.max(1, watermark - smallest); + while (true) { + ensureCapacity(candidate + largest); + boolean fits = true; + for (int k = 0; fits && k < labelCount; k++) { + fits = check[candidate + labels[k]] == EMPTY; + } + if (fits) { + while (watermark < check.length && check[watermark] != EMPTY) { + watermark++; + } + return candidate; + } + candidate++; + } } - Arrays.sort(keys); - final TrieNode[] nodes = new TrieNode[keys.length]; - for (int k = 0; k < keys.length; k++) { - nodes[k] = children.get(keys[k]).freeze(); + + private void ensureCapacity(int slot) { + if (slot >= check.length) { + int capacity = check.length; + while (capacity <= slot) { + capacity += capacity >> 1; + } + base = Arrays.copyOf(base, capacity); + final int old = check.length; + check = Arrays.copyOf(check, capacity); + Arrays.fill(check, old, capacity, EMPTY); + } } - return new TrieNode(keys, nodes, entries); } } @@ -291,7 +487,7 @@ interface PrefixMatchConsumer { void accept(int length, List entries); } - private final TrieNode lexicon; + private final DoubleArrayLexicon lexicon; private final int maxSurfaceLength; private final short[] connectionCosts; private final int rightSize; @@ -300,7 +496,7 @@ interface PrefixMatchConsumer { private final Category defaultCategory; private final Map> unknownEntries; - private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, + private MecabDictionary(DoubleArrayLexicon lexicon, int maxSurfaceLength, short[] connectionCosts, int rightSize, Map categories, CategoryTable categoryTable, Map> unknownEntries) { this.lexicon = lexicon; @@ -409,23 +605,8 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc final Map> unknown = new HashMap<>(); readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); - return new MecabDictionary(buildTrie(lexicon), maxSurface, costs, rightSize, - categories, categoryTable.build(categories), unknown); - } - - /** Folds the surface-keyed lexicon into a character trie for prefix search. */ - private static TrieNode buildTrie(Map> lexicon) { - final TrieBuilderNode root = new TrieBuilderNode(); - for (final Map.Entry> entry : lexicon.entrySet()) { - TrieBuilderNode node = root; - final String surface = entry.getKey(); - for (int i = 0; i < surface.length(); i++) { - node = node.children.computeIfAbsent(surface.charAt(i), - key -> new TrieBuilderNode()); - } - node.entries = List.copyOf(entry.getValue()); - } - return root.freeze(); + return new MecabDictionary(DoubleArrayLexicon.build(lexicon), maxSurface, costs, + rightSize, categories, categoryTable.build(categories), unknown); } /** @@ -534,11 +715,7 @@ private static void readCharacterDefinition(Path file, Charset charset, * @return The entries, or {@code null} when the surface is not listed. */ List lookup(String surface) { - TrieNode node = lexicon; - for (int i = 0; i < surface.length() && node != null; i++) { - node = node.child(surface.charAt(i)); - } - return node == null ? null : node.entries; + return lexicon.lookup(surface); } /** @@ -551,16 +728,7 @@ List lookup(String surface) { * @param consumer Receives each match. */ void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) { - TrieNode node = lexicon; - for (int i = from; i < to; i++) { - node = node.child(text.charAt(i)); - if (node == null) { - return; - } - if (node.entries != null) { - consumer.accept(i - from + 1, node.entries); - } - } + lexicon.prefixMatches(text, from, to, consumer); } /** @return The length in characters of the longest surface form in the lexicon. */ From 19c58d2652101a49181bdb741a7213836044bbdd Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 07:58:38 -0400 Subject: [PATCH 10/14] OPENNLP-1894: Chain lattice nodes intrusively instead of allocating per-position lists The Viterbi lattice held one ArrayList per text position plus one fresh candidate list per position, pure allocation churn on long stretches. Nodes ending at a position now chain through their own link field behind a single head reference per position, and candidate gathering fills one scratch list reused across positions, so building the lattice allocates nothing besides the nodes themselves. IPADIC throughput on the 400k-character harness rises from 5.6M to 6.5M characters per second with identical output. --- .../tokenize/lattice/LatticeTokenizer.java | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index f7a479bf83..98f75d1b6b 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -67,7 +67,11 @@ public LatticeTokenizer(MecabDictionary dictionary) { this.dictionary = dictionary; } - /** One lattice node: a candidate morpheme with its best path cost so far. */ + /** + * One lattice node: a candidate morpheme with its best path cost so far. Nodes + * ending at one position chain through {@link #nextEndingHere}, so the lattice + * needs one head reference per position instead of a list allocation. + */ private static final class Node { private final int start; private final int end; @@ -75,6 +79,7 @@ private static final class Node { private final boolean unknown; private long pathCost = Long.MAX_VALUE; private Node previous; + private Node nextEndingHere; private Node(int start, int end, WordEntry entry, boolean unknown) { this.start = start; @@ -136,12 +141,9 @@ public Span[] tokenizePos(String s) { /** Runs the Viterbi search over one whitespace-free stretch of text. */ private void decode(String text, int from, int to, List morphemes) { final int length = to - from; - final List> endingAt = new ArrayList<>(length + 1); - for (int i = 0; i <= length; i++) { - endingAt.add(new ArrayList<>()); - } - final boolean[] reachable = new boolean[length + 1]; - reachable[0] = true; + // One head reference per position; nodes ending there chain through the node's + // own link, so building the lattice allocates nothing besides the nodes. + final Node[] endingAt = new Node[length + 1]; // One right-to-left pass fixes each position's category and same-category run end, // so candidate generation reads them instead of rescanning the run from every @@ -150,23 +152,25 @@ private void decode(String text, int from, int to, List morphemes) { final int[] runEndAt = new int[length]; computeCategoryRuns(text, from, to, categoryAt, runEndAt); + final List candidates = new ArrayList<>(); for (int i = 0; i < length; i++) { - if (!reachable[i]) { + if (i > 0 && endingAt[i] == null) { continue; } - final List candidates = - candidates(text, from, to, i, categoryAt[i], runEndAt[i]); + candidates.clear(); + candidates(text, from, to, i, categoryAt[i], runEndAt[i], candidates); for (final Node candidate : candidates) { - relax(candidate, i == 0 ? null : endingAt.get(i)); + relax(candidate, i == 0 ? null : endingAt[i]); if (candidate.pathCost < Long.MAX_VALUE) { - endingAt.get(candidate.end - from).add(candidate); - reachable[candidate.end - from] = true; + final int end = candidate.end - from; + candidate.nextEndingHere = endingAt[end]; + endingAt[end] = candidate; } } } Node best = null; - for (final Node node : endingAt.get(length)) { + for (Node node = endingAt[length]; node != null; node = node.nextEndingHere) { final long total = node.pathCost + dictionary.connectionCost(node.entry.rightId(), BOUNDARY_CONTEXT); if (best == null || total < best.pathCost @@ -190,13 +194,14 @@ private void decode(String text, int from, int to, List morphemes) { } /** Connects a candidate to the cheapest predecessor ending where it starts. */ - private void relax(Node candidate, List predecessors) { + private void relax(Node candidate, Node predecessors) { if (predecessors == null) { candidate.pathCost = candidate.entry.cost() + dictionary.connectionCost(BOUNDARY_CONTEXT, candidate.entry.leftId()); return; } - for (final Node predecessor : predecessors) { + for (Node predecessor = predecessors; predecessor != null; + predecessor = predecessor.nextEndingHere) { final long total = predecessor.pathCost + dictionary.connectionCost(predecessor.entry.rightId(), candidate.entry.leftId()) + candidate.entry.cost(); @@ -238,10 +243,9 @@ private void computeCategoryRuns(String text, int from, int to, } /** Gathers lexicon matches and unknown-word candidates starting at one position. */ - private List candidates(String text, int from, int to, int offset, - Category positionCategory, int positionRunEnd) { + private void candidates(String text, int from, int to, int offset, + Category positionCategory, int positionRunEnd, List candidates) { final int position = from + offset; - final List candidates = new ArrayList<>(); final boolean[] matched = new boolean[1]; dictionary.prefixMatches(text, position, to, (length, entries) -> { matched[0] = true; @@ -285,7 +289,6 @@ private List candidates(String text, int from, int to, int offset, throw new IllegalStateException("dictionary provides no candidate at position " + position + "; unk.def lacks a DEFAULT template"); } - return candidates; } /** From 7536cf13b6177cb3df1bde3caa7244f6ba3ffaee Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:44:21 -0400 Subject: [PATCH 11/14] OPENNLP-1894: Document lattice CJK tokenization with a mirror-tested example Add a lattice tokenizer section to the manual citing LatticeUsageExampleTest. --- opennlp-docs/src/docbkx/tokenizer.xml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index cd1d8a2ddf..fbc74fbb2e 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -538,5 +538,30 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { +
+ Lattice tokenization for CJK + + Languages written without spaces need a dictionary-backed segmenter. + LatticeTokenizer scores paths over a mecab-format dictionary and + emits the cheapest segmentation with spans in original text coordinates. + UnigramSegmenter does the same from a plain frequency lexicon. + Install a dictionary archive with MecabDictionaryInstaller, load it + as a MecabDictionary, and tokenize. + LatticeUsageExampleTest asserts the install-load-tokenize and + lexicon flows shown here. + + morphemes = tokenizer.analyze(text); + +UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt")); +String[] words = segmenter.tokenize(text);]]> + + +
From e689f3f8d5f351abe32c5d7ad139973dda057bc2 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 21 Jul 2026 06:39:29 -0400 Subject: [PATCH 12/14] OPENNLP-1894: Apply the review-convention pass and drop unreferenced lexicon accessors --- .../dev => dev}/README-mecab-dictionaries.md | 10 +- .../dev => dev}/download-mecab-dictionary.sh | 4 +- .../tokenize/lattice/LatticeTokenizer.java | 13 +-- .../tokenize/lattice/MecabDictionary.java | 93 ++++-------------- .../lattice/MecabDictionaryInstaller.java | 22 +++-- .../tokenize/lattice/UnigramSegmenter.java | 19 ++-- .../lattice/LatticeTokenizerTest.java | 6 +- .../lattice/LatticeUsageExampleTest.java | 68 +------------ .../lattice/MecabDictionaryInstallerTest.java | 55 ++--------- .../tools/tokenize/lattice/TarGzArchives.java | 97 +++++++++++++++++++ .../lattice/UnigramSegmenterTest.java | 8 +- opennlp-docs/src/docbkx/tokenizer.xml | 2 +- 12 files changed, 171 insertions(+), 226 deletions(-) rename {opennlp-core/opennlp-runtime/dev => dev}/README-mecab-dictionaries.md (88%) rename {opennlp-core/opennlp-runtime/dev => dev}/download-mecab-dictionary.sh (94%) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java diff --git a/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md b/dev/README-mecab-dictionaries.md similarity index 88% rename from opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md rename to dev/README-mecab-dictionaries.md index c3244ba9c4..fce653b397 100644 --- a/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md +++ b/dev/README-mecab-dictionaries.md @@ -17,7 +17,7 @@ # CJK dictionaries for the lattice tokenizer -The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a mecab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and by using it you accept that dictionary's license. Read the license file inside the archive before use; nothing in this repository grants you those terms. +The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a mecab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and each dictionary carries its own license. Read the license file inside the archive before use. ## Known mecab-format dictionary projects @@ -63,7 +63,8 @@ import opennlp.tools.tokenize.lattice.MecabDictionary; MecabDictionary dictionary = MecabDictionary.load(Path.of("ipadic"), Charset.forName("EUC-JP")); LatticeTokenizer tokenizer = new LatticeTokenizer(dictionary); -String[] tokens = tokenizer.tokenize("東京都に行く"); +// "Tokyo-to ni iku" (go to the Tokyo metropolis), escaped to keep this file ASCII +String[] tokens = tokenizer.tokenize("\u6771\u4EAC\u90FD\u306B\u884C\u304F"); ``` For a UTF-8 dictionary such as mecab-ko-dic, `MecabDictionary.load(Path.of("ko-dic"))` is enough. Loaded dictionaries and tokenizers are immutable and safe to share between threads, so load once and reuse. @@ -77,7 +78,8 @@ import java.nio.file.Path; import opennlp.tools.tokenize.lattice.UnigramSegmenter; UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt")); -String[] tokens = segmenter.tokenize("我来到北京天安门"); +// "wo laidao Beijing Tian'anmen" (I arrive at Beijing Tiananmen), escaped as above +String[] tokens = segmenter.tokenize("\u6211\u6765\u5230\u5317\u4EAC\u5929\u5B89\u95E8"); ``` -As with the dictionaries, the lexicon's license is between you and its publisher; nothing is bundled. +As with the dictionaries, the lexicon carries its own license; nothing is bundled. diff --git a/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh b/dev/download-mecab-dictionary.sh similarity index 94% rename from opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh rename to dev/download-mecab-dictionary.sh index 30c5b23bde..3b86ed09d6 100755 --- a/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh +++ b/dev/download-mecab-dictionary.sh @@ -23,8 +23,8 @@ # this directory for the known dictionary projects, their encodings, and the Java # steps that follow the download. # -# The dictionary you download carries its own license, which you accept by using it. -# Nothing is bundled with Apache OpenNLP and no dictionary location is built in. +# The dictionary you download carries its own license. Nothing is bundled with +# Apache OpenNLP and no dictionary location is built in. set -euo pipefail diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index 98f75d1b6b..5ee2384ba1 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -69,8 +69,7 @@ public LatticeTokenizer(MecabDictionary dictionary) { /** * One lattice node: a candidate morpheme with its best path cost so far. Nodes - * ending at one position chain through {@link #nextEndingHere}, so the lattice - * needs one head reference per position instead of a list allocation. + * ending at one position chain through {@link #nextEndingHere}. */ private static final class Node { private final int start; @@ -141,13 +140,9 @@ public Span[] tokenizePos(String s) { /** Runs the Viterbi search over one whitespace-free stretch of text. */ private void decode(String text, int from, int to, List morphemes) { final int length = to - from; - // One head reference per position; nodes ending there chain through the node's - // own link, so building the lattice allocates nothing besides the nodes. + // Each element heads the chain of nodes ending at that position. final Node[] endingAt = new Node[length + 1]; - // One right-to-left pass fixes each position's category and same-category run end, - // so candidate generation reads them instead of rescanning the run from every - // position, which would cost the square of the run length on long uniform runs. final Category[] categoryAt = new Category[length]; final int[] runEndAt = new int[length]; computeCategoryRuns(text, from, to, categoryAt, runEndAt); @@ -259,9 +254,9 @@ private void candidates(String text, int from, int to, int offset, final Category category; final int runEnd; if (positionCategory == null) { - // Only a lexicon surface ending inside a surrogate pair could make such a + // Only a lexicon surface ending inside a surrogate pair can make such a // position reachable; classify the stray code unit on the spot so the lattice - // stays connected the way it always did. + // stays connected. category = dictionary.categoryOf(codePoint); runEnd = position + Character.charCount(codePoint); } else { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index cd1f46ba7f..ed0775a19e 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -35,9 +35,8 @@ * An immutable, in-memory dictionary in the mecab directory format: lexicon entries * from the {@code *.csv} files, connection costs from {@code matrix.def}, character * categories from {@code char.def}, and unknown-word templates from {@code unk.def}, - * loaded from a user-supplied dictionary directory. The reader is a clean-room - * implementation of the documented format; no dictionary data is bundled or downloaded - * by this class, so the dictionaries' own licenses never attach to this library. + * loaded from a user-supplied dictionary directory. No dictionary data is bundled or + * downloaded by this class. * *

The same format serves multiple languages: the Japanese IPADIC and UniDic * distributions and the Korean mecab-ko-dic all load through this one reader, with the @@ -60,12 +59,9 @@ record Category(String name, boolean invoke, boolean group, int length) { /** * The lexicon as a double-array trie: one transition is one array read and one - * comparison, so the per-position prefix walk of the tokenizer never hashes, never - * binary-searches a fan-out, and never boxes. Characters are recoded into dense - * labels ordered by frequency before the array is built, which keeps the array - * compact although CJK surfaces draw from tens of thousands of distinct - * characters; a character the lexicon never uses misses in the recode table before - * the array is even consulted. + * comparison. Characters are recoded into dense labels ordered by descending + * frequency before the array is built, which keeps the array compact; a character + * the lexicon never uses misses in the recode table before the array is consulted. * *

The layout is the classic base/check pair: from state {@code s}, label * {@code c} leads to {@code t = base[s] + c} exactly when {@code check[t] == s}. @@ -160,32 +156,6 @@ private void prefixMatches(String text, int from, int to, } } - /** - * Looks up the entries of an exact surface form. - * - * @param surface The surface form. - * @return The entries, or {@code null} when the surface is not listed. - */ - private List lookup(String surface) { - int state = Builder.ROOT; - for (int i = 0; i < surface.length(); i++) { - final int code = codeOf[surface.charAt(i)]; - if (code < 0) { - return null; - } - final int next = base[state] + code; - if (next >= check.length || check[next] != state) { - return null; - } - state = next; - } - final int terminal = base[state]; - if (terminal < check.length && check[terminal] == state && base[terminal] < 0) { - return values[-base[terminal] - 1]; - } - return null; - } - /** * The recursive sorted-range builder: each call places one node's children by * finding a base at which every child label lands on a free slot, then recurses @@ -310,12 +280,9 @@ private void ensureCapacity(int slot) { * The {@code char.def} code point to category name mapping, over the whole Unicode * code point range. * - *

The Basic Multilingual Plane is held in a directly indexed array, which is one - * reference per BMP code point and is the entire mapping for a BMP-only dictionary. - * The supplementary planes are held as a sorted, non-overlapping range table searched - * by binary search, because dictionaries map them in a handful of large blocks: a - * directly indexed array over all of Unicode would spend more than four megabytes of - * references to say what a few range rows already say.

+ *

The Basic Multilingual Plane is held in a directly indexed array. The + * supplementary planes are held as a sorted, non-overlapping range table searched by + * binary search, because dictionaries map them in a handful of large blocks.

*/ private static final class CategoryTable { @@ -334,10 +301,8 @@ private CategoryTable(Category[] bmp, int[] rangeStart, int[] rangeEnd, /** * Looks up the category a {@code char.def} mapping gives a code point. The table - * holds the {@link Category} instances themselves, so a lookup on the tokenizer's - * per-character path costs one array read or one binary search, never a name map - * access, and two code points of one category share one instance to compare by - * identity. + * holds the {@link Category} instances themselves, and two code points of one + * category share one instance, so categories may be compared by identity. * * @param codePoint The code point to classify. * @return The category, or {@code null} when no mapping covers the code point. @@ -488,7 +453,6 @@ interface PrefixMatchConsumer { } private final DoubleArrayLexicon lexicon; - private final int maxSurfaceLength; private final short[] connectionCosts; private final int rightSize; private final Map categories; @@ -496,11 +460,10 @@ interface PrefixMatchConsumer { private final Category defaultCategory; private final Map> unknownEntries; - private MecabDictionary(DoubleArrayLexicon lexicon, int maxSurfaceLength, + private MecabDictionary(DoubleArrayLexicon lexicon, short[] connectionCosts, int rightSize, Map categories, CategoryTable categoryTable, Map> unknownEntries) { this.lexicon = lexicon; - this.maxSurfaceLength = maxSurfaceLength; this.connectionCosts = connectionCosts; this.rightSize = rightSize; this.categories = categories; @@ -536,8 +499,11 @@ public static MecabDictionary load(Path directory) throws IOException { * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static MecabDictionary load(Path directory, Charset charset) throws IOException { - if (directory == null || charset == null) { - throw new IllegalArgumentException("directory and charset must not be null"); + if (directory == null) { + throw new IllegalArgumentException("directory must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); } // The connection matrix is read first because its dimensions are what every // lexicon entry's context ids have to be inside of. @@ -587,11 +553,9 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } final Map> lexicon = new HashMap<>(); - int maxSurface = 0; try (DirectoryStream csvFiles = Files.newDirectoryStream(directory, "*.csv")) { for (final Path csv : csvFiles) { - maxSurface = Math.max(maxSurface, - readLexicon(csv, charset, lexicon, leftSize, rightSize)); + readLexicon(csv, charset, lexicon, leftSize, rightSize); } } if (lexicon.isEmpty()) { @@ -605,7 +569,7 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc final Map> unknown = new HashMap<>(); readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); - return new MecabDictionary(DoubleArrayLexicon.build(lexicon), maxSurface, costs, + return new MecabDictionary(DoubleArrayLexicon.build(lexicon), costs, rightSize, categories, categoryTable.build(categories), unknown); } @@ -620,14 +584,12 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc * ids. * @param rightSize The second {@code matrix.def} dimension, which bounds left context * ids. - * @return The length in characters of the longest surface form read. * @throws IOException Thrown if the file is missing, an entry is malformed, or an * entry's context id is outside the matrix dimensions. */ - private static int readLexicon(Path file, Charset charset, + private static void readLexicon(Path file, Charset charset, Map> target, int leftSize, int rightSize) throws IOException { - int maxSurface = 0; int lineNumber = 0; for (final String line : readLines(file, charset)) { lineNumber++; @@ -658,9 +620,7 @@ private static int readLexicon(Path file, Charset charset, parseInt(fields.get(3), file.toString(), lineNumber), List.copyOf(fields.subList(4, fields.size()))); target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); - maxSurface = Math.max(maxSurface, surface.length()); } - return maxSurface; } /** Reads char.def: category behavior lines and code point mapping lines. */ @@ -708,16 +668,6 @@ private static void readCharacterDefinition(Path file, Charset charset, } } - /** - * Looks up the lexicon entries for an exact surface form. - * - * @param surface The surface form. - * @return The entries, or {@code null} when the surface is not listed. - */ - List lookup(String surface) { - return lexicon.lookup(surface); - } - /** * Reports every lexicon surface starting at a text position, walking the trie once * with no substring allocation. @@ -731,11 +681,6 @@ void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) lexicon.prefixMatches(text, from, to, consumer); } - /** @return The length in characters of the longest surface form in the lexicon. */ - int maxSurfaceLength() { - return maxSurfaceLength; - } - /** * Reads the connection cost between two adjacent nodes. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 9bbbde7902..3c2589bc02 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -29,9 +29,8 @@ /** * Fetches and unpacks a mecab-format dictionary archive into a local directory, so the * dictionary is acquired by the user at install time and never ships with this library. - * The user supplies the archive location from the dictionary project of their choice - * and thereby accepts that dictionary's license; nothing is bundled and no location is - * built in. + * The archive location is user-supplied; no dictionary data is bundled and no download + * location is built in. * *

The installer reads gzip-compressed tar archives, the format the common * distributions use, and extracts only the dictionary payload: the {@code *.csv} @@ -64,11 +63,15 @@ private MecabDictionaryInstaller() { * @return The number of dictionary files extracted. * @throws IOException Thrown if fetching, reading, or writing fails, or the archive * contains no dictionary file. - * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or + * {@code archive} is not an absolute URI. */ public static int install(URI archive, Path targetDirectory) throws IOException { - if (archive == null || targetDirectory == null) { - throw new IllegalArgumentException("archive and targetDirectory must not be null"); + if (archive == null) { + throw new IllegalArgumentException("archive must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); } try (InputStream in = archive.toURL().openStream()) { return extract(in, targetDirectory); @@ -89,8 +92,11 @@ public static int install(URI archive, Path targetDirectory) throws IOException */ public static int extract(InputStream archiveStream, Path targetDirectory) throws IOException { - if (archiveStream == null || targetDirectory == null) { - throw new IllegalArgumentException("stream and targetDirectory must not be null"); + if (archiveStream == null) { + throw new IllegalArgumentException("archiveStream must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); } Files.createDirectories(targetDirectory); final InputStream tar = new GZIPInputStream(archiveStream); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index b4cfaa8ea6..ba3e06861f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -41,9 +41,8 @@ * and counts. * *

The lexicon format is one entry per line: the word, its count, and optionally a - * tag, separated by whitespace. The user supplies the lexicon file and thereby accepts - * its license; nothing is bundled. Every reported span is in original text - * coordinates.

+ * tag, separated by whitespace. The lexicon file is user-supplied; no lexicon data is + * bundled. Every reported span is in original text coordinates.

* *

Instances are immutable and safe to share between threads.

* @@ -90,8 +89,11 @@ public static UnigramSegmenter load(Path lexicon) throws IOException { * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static UnigramSegmenter load(Path lexicon, Charset charset) throws IOException { - if (lexicon == null || charset == null) { - throw new IllegalArgumentException("lexicon and charset must not be null"); + if (lexicon == null) { + throw new IllegalArgumentException("lexicon must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); } try (InputStream in = Files.newInputStream(lexicon)) { return load(in, charset); @@ -109,8 +111,11 @@ public static UnigramSegmenter load(Path lexicon, Charset charset) throws IOExce */ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) throws IOException { - if (lexiconStream == null || charset == null) { - throw new IllegalArgumentException("stream and charset must not be null"); + if (lexiconStream == null) { + throw new IllegalArgumentException("lexiconStream must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); } final Map counts = new HashMap<>(); long total = 0; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index e1be5a4a98..034cd6d8c3 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -99,7 +99,7 @@ void testMorphemesCarryDictionaryFeatures() { Assertions.assertEquals(4, morphemes.size()); Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); Assertions.assertEquals(List.of("particle", "case"), morphemes.get(2).features()); - Assertions.assertEquals(false, morphemes.get(0).unknown()); + Assertions.assertFalse(morphemes.get(0).unknown()); } @Test @@ -107,7 +107,7 @@ void testUnknownLatinRunGroupsIntoOneMorpheme() { final List morphemes = tokenizer.analyze("ABC\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); Assertions.assertEquals("ABC", morphemes.get(0).surface()); - Assertions.assertEquals(true, morphemes.get(0).unknown()); + Assertions.assertTrue(morphemes.get(0).unknown()); Assertions.assertEquals(List.of("noun", "foreign"), morphemes.get(0).features()); } @@ -116,7 +116,7 @@ void testUnknownKanjiPreferOneMorphemeOverTwo() { final List morphemes = tokenizer.analyze("\u5CE0\u9053\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); Assertions.assertEquals("\u5CE0\u9053", morphemes.get(0).surface()); - Assertions.assertEquals(true, morphemes.get(0).unknown()); + Assertions.assertTrue(morphemes.get(0).unknown()); } /** diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java index a61969c902..0fc58585de 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -17,14 +17,12 @@ package opennlp.tools.tokenize.lattice; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.zip.GZIPOutputStream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -50,70 +48,6 @@ */ public class LatticeUsageExampleTest { - /** - * Appends one ustar file entry to a growing tar image: a 512-byte header block - * followed by the content padded to a block boundary. - * - * @param tar The tar image under construction. Must not be {@code null}. - * @param name The entry name including any directory prefix. Must not be - * {@code null}, empty, or longer than the 100-byte header name field. - * @param content The entry content bytes. Must not be {@code null}. - * @throws IOException Thrown if writing to the in-memory stream fails. - * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. - */ - private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) - throws IOException { - final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); - if (nameBytes.length == 0 || nameBytes.length > 100) { - throw new IllegalArgumentException( - "entry name must be 1..100 bytes, got " + nameBytes.length); - } - final byte[] header = new byte[512]; - System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); - final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); - System.arraycopy(mode, 0, header, 100, mode.length); - final String size = String.format("%011o", content.length); - System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); - header[156] = '0'; - for (int i = 148; i < 156; i++) { - header[i] = ' '; - } - int checksum = 0; - for (final byte b : header) { - checksum += b & 0xFF; - } - final String checksumText = String.format("%06o", checksum); - System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); - header[154] = 0; - header[155] = ' '; - tar.write(header); - tar.write(content); - final int padding = (512 - content.length % 512) % 512; - tar.write(new byte[padding]); - } - - /** - * Builds a gzip-compressed tar archive from name and content pairs, the layout a - * dictionary distribution ships in. - * - * @param entries The entries as {@code {name, content}} pairs. Must not be - * {@code null}. - * @return The compressed archive bytes. Never {@code null}. - * @throws IOException Thrown if writing to the in-memory streams fails. - */ - private static byte[] gzippedTar(String[][] entries) throws IOException { - final ByteArrayOutputStream tar = new ByteArrayOutputStream(); - for (final String[] entry : entries) { - tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); - } - tar.write(new byte[1024]); - final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); - try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { - gzip.write(tar.toByteArray()); - } - return compressed.toByteArray(); - } - /** * Walks the full mecab-format flow: package a miniature Japanese dictionary as a * {@code tar.gz} archive, install it from a file URI, load it, and tokenize. The @@ -126,7 +60,7 @@ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) throws IOException { // A minimal but complete dictionary: one lexicon file plus the three definition // files every mecab-format distribution contains, wrapped like a release archive. - final byte[] archive = gzippedTar(new String[][] { + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { {"mini-dict-0.1/lexicon.csv", String.join("\n", "\u6771\u4EAC,0,0,3000,noun,proper", "\u4EAC\u90FD,0,0,3000,noun,proper", diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java index fe999e4609..da95c97366 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -18,64 +18,24 @@ package opennlp.tools.tokenize.lattice; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.zip.GZIPOutputStream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +/** + * Tests the installer against project-authored, in-memory archives; no external + * dictionary data and no network access are involved. + */ public class MecabDictionaryInstallerTest { - /** Writes one ustar entry: a 512-byte header block and padded content. */ - private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) - throws IOException { - final byte[] header = new byte[512]; - final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); - System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); - final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); - System.arraycopy(mode, 0, header, 100, mode.length); - final String size = String.format("%011o", content.length); - System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); - header[156] = '0'; - for (int i = 148; i < 156; i++) { - header[i] = ' '; - } - int checksum = 0; - for (final byte b : header) { - checksum += b & 0xFF; - } - final String checksumText = String.format("%06o", checksum); - System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); - header[154] = 0; - header[155] = ' '; - tar.write(header); - tar.write(content); - final int padding = (512 - content.length % 512) % 512; - tar.write(new byte[padding]); - } - - private static byte[] archive(String[][] entries) throws IOException { - final ByteArrayOutputStream tar = new ByteArrayOutputStream(); - for (final String[] entry : entries) { - tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); - } - tar.write(new byte[1024]); - final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); - try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { - gzip.write(tar.toByteArray()); - } - return compressed.toByteArray(); - } - @Test void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target) throws IOException { - final byte[] archive = archive(new String[][] { + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { {"dict-1.0/lexicon.csv", "cat,0,0,100,noun\n"}, {"dict-1.0/matrix.def", "1 1\n0 0 0\n"}, {"dict-1.0/char.def", "DEFAULT 0 1 0\n"}, @@ -101,7 +61,7 @@ void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target) void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target) throws IOException { final Path archiveFile = source.resolve("dict.tar.gz"); - Files.write(archiveFile, archive(new String[][] { + Files.write(archiveFile, TarGzArchives.gzippedTar(new String[][] { {"d/words.csv", "cat,0,0,100,noun\n"}, {"d/matrix.def", "1 1\n0 0 0\n"}})); @@ -115,7 +75,8 @@ void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target) @Test void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target) throws IOException { - final byte[] archive = archive(new String[][] {{"readme.txt", "nothing here"}}); + final byte[] archive = + TarGzArchives.gzippedTar(new String[][] {{"readme.txt", "nothing here"}}); Assertions.assertThrows(IOException.class, () -> MecabDictionaryInstaller.extract( new ByteArrayInputStream(archive), target)); } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java new file mode 100644 index 0000000000..4fdfb29d01 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPOutputStream; + +/** + * Builds miniature, project-authored gzip-compressed ustar archives in memory for the + * tests of this package; no external archive data is involved. + */ +final class TarGzArchives { + + private TarGzArchives() { + } + + /** + * Builds a gzip-compressed tar archive from name and content pairs, the layout a + * dictionary distribution ships in. + * + * @param entries The entries as {@code {name, content}} pairs. Must not be + * {@code null}. + * @return The compressed archive bytes. Never {@code null}. + * @throws IOException Thrown if writing to the in-memory streams fails. + */ + static byte[] gzippedTar(String[][] entries) throws IOException { + final ByteArrayOutputStream tar = new ByteArrayOutputStream(); + for (final String[] entry : entries) { + tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); + } + tar.write(new byte[1024]); + final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { + gzip.write(tar.toByteArray()); + } + return compressed.toByteArray(); + } + + /** + * Appends one ustar file entry to a growing tar image: a 512-byte header block + * followed by the content padded to a block boundary. + * + * @param tar The tar image under construction. Must not be {@code null}. + * @param name The entry name including any directory prefix. Must not be + * {@code null}, empty, or longer than the 100-byte header name field. + * @param content The entry content bytes. Must not be {@code null}. + * @throws IOException Thrown if writing to the in-memory stream fails. + * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. + */ + private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) + throws IOException { + final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + if (nameBytes.length == 0 || nameBytes.length > 100) { + throw new IllegalArgumentException( + "entry name must be 1..100 bytes, got " + nameBytes.length); + } + final byte[] header = new byte[512]; + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(mode, 0, header, 100, mode.length); + final String size = String.format("%011o", content.length); + System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); + header[156] = '0'; + for (int i = 148; i < 156; i++) { + header[i] = ' '; + } + int checksum = 0; + for (final byte b : header) { + checksum += b & 0xFF; + } + final String checksumText = String.format("%06o", checksum); + System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); + header[154] = 0; + header[155] = ' '; + tar.write(header); + tar.write(content); + final int padding = (512 - content.length % 512) % 512; + tar.write(new byte[padding]); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index e6ebd604e1..f5cff587bc 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -183,19 +183,19 @@ void testInvalidArguments() { * span over both of its surrogate halves, and no span boundary ever lands between * them. */ - @org.junit.jupiter.api.Test + @Test void testUnknownSupplementaryCharacterIsNeverSplit() { // U+20BB7, a CJK extension B ideograph, written as its surrogate pair final String text = "\uD842\uDFB7\uD842\uDFB7"; - final opennlp.tools.util.Span[] spans = segmenter.tokenizePos(text); - for (final opennlp.tools.util.Span span : spans) { + final Span[] spans = segmenter.tokenizePos(text); + for (final Span span : spans) { Assertions.assertEquals(0, span.getStart() % 2, "span must start on a code point boundary: " + span); Assertions.assertEquals(0, span.getEnd() % 2, "span must end on a code point boundary: " + span); } int covered = 0; - for (final opennlp.tools.util.Span span : spans) { + for (final Span span : spans) { covered += span.length(); } Assertions.assertEquals(text.length(), covered); diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index fbc74fbb2e..6f74b6a601 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -542,7 +542,7 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { Lattice tokenization for CJK Languages written without spaces need a dictionary-backed segmenter. - LatticeTokenizer scores paths over a mecab-format dictionary and + LatticeTokenizer scores paths over a MeCab-format dictionary and emits the cheapest segmentation with spans in original text coordinates. UnigramSegmenter does the same from a plain frequency lexicon. Install a dictionary archive with MecabDictionaryInstaller, load it From ceca4883246109884e6cb4c7e2fce50b24cd0aa3 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 24 Jul 2026 14:53:05 -0400 Subject: [PATCH 13/14] OPENNLP-1894: Trim parsed lines as Unicode whitespace and document the tokenizer overrides The frequency lexicon was trimmed with String.trim(), which strips only ASCII control characters and the space. A line starting with an ideographic space (U+3000), ordinary in hand-edited CJK text files, therefore kept that space as part of the word and pushed the count field one token to the right, so the load failed as a malformed count. The lexicon reader now trims with StringUtil.trimUnicodeWhitespace, matching the White_Space convention the rest of the tokenizer already scans by, and a test pins the leading U+3000 case. The mecab reader's line and numeric-field trims move to the same call so one class does not mix two whitespace judgments; those fields are ASCII in valid dictionaries, so the behavior there is unchanged. Both tokenizer views also gain {@inheritDoc} and their null contract, and the unknown-candidate helper drops a static modifier it did not need. --- .../tools/tokenize/lattice/LatticeTokenizer.java | 16 +++++++++++++++- .../tools/tokenize/lattice/MecabDictionary.java | 8 ++++---- .../tools/tokenize/lattice/UnigramSegmenter.java | 16 +++++++++++++++- .../tokenize/lattice/UnigramSegmenterTest.java | 15 +++++++++++++++ 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index 5ee2384ba1..7ce8be2aa4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -117,6 +117,13 @@ public List analyze(String text) { return morphemes; } + /** + * {@inheritDoc} + * + *

Reports the segmented surfaces, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + */ @Override public String[] tokenize(String s) { final List morphemes = analyze(s); @@ -127,6 +134,13 @@ public String[] tokenize(String s) { return tokens; } + /** + * {@inheritDoc} + * + *

Reports the segmented spans in original text coordinates, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + */ @Override public Span[] tokenizePos(String s) { final List morphemes = analyze(s); @@ -301,7 +315,7 @@ private void candidates(String text, int from, int to, int offset, * @param category The category of that run. * @param templates The category's unknown-word templates. */ - private static void addUnknown(List candidates, String text, int position, + private void addUnknown(List candidates, String text, int position, int runEnd, Category category, List templates) { if (category.group()) { for (final WordEntry entry : templates) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index ed0775a19e..a971c586a3 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -528,7 +528,7 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } final short[] costs = new short[(int) cells]; for (int i = 1; i < matrixLines.size(); i++) { - final String line = matrixLines.get(i).trim(); + final String line = StringUtil.trimUnicodeWhitespace(matrixLines.get(i)); if (line.isEmpty()) { continue; } @@ -630,7 +630,7 @@ private static void readCharacterDefinition(Path file, Charset charset, int lineNumber = 0; for (final String raw : readLines(file, charset)) { lineNumber++; - final String line = stripComment(raw).trim(); + final String line = StringUtil.trimUnicodeWhitespace(stripComment(raw)); if (line.isEmpty()) { continue; } @@ -809,7 +809,7 @@ private static String[] splitWhitespace(String line) { private static int parseInt(String text, String file, int lineNumber) throws IOException { try { - return Integer.parseInt(text.trim()); + return Integer.parseInt(StringUtil.trimUnicodeWhitespace(text)); } catch (NumberFormatException e) { throw new IOException("malformed number in " + file + " line " + lineNumber, e); } @@ -829,7 +829,7 @@ private static int parseCodePoint(String text, Path file, int lineNumber) throws IOException { final int codePoint; try { - codePoint = Integer.parseInt(text.trim().substring(2), 16); + codePoint = Integer.parseInt(StringUtil.trimUnicodeWhitespace(text).substring(2), 16); } catch (RuntimeException e) { throw new IOException("malformed code point in " + file + " line " + lineNumber, e); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index ba3e06861f..00469ac4f7 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -127,7 +127,7 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) continue; } lineNumber++; - final String line = content.substring(lineStart, i).trim(); + final String line = StringUtil.trimUnicodeWhitespace(content.substring(lineStart, i)); lineStart = i + 1; if (line.isEmpty()) { continue; @@ -178,6 +178,13 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) return new UnigramSegmenter(root, unknown); } + /** + * {@inheritDoc} + * + *

Reports the segmented surfaces, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + */ @Override public String[] tokenize(String s) { final Span[] spans = tokenizePos(s); @@ -188,6 +195,13 @@ public String[] tokenize(String s) { return tokens; } + /** + * {@inheritDoc} + * + *

Reports the segmented spans in original text coordinates, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + */ @Override public Span[] tokenizePos(String s) { if (s == null) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index f5cff587bc..977d42fcc4 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -200,4 +200,19 @@ void testUnknownSupplementaryCharacterIsNeverSplit() { } Assertions.assertEquals(text.length(), covered); } + + /** + * Pins Unicode-whitespace trimming of lexicon lines: a leading ideographic space + * (U+3000), common in hand-edited CJK text files, is stripped like ASCII whitespace, + * so the entry loads rather than failing as a malformed count. + */ + @Test + void testLeadingIdeographicSpaceIsTrimmed() throws IOException { + // U+3000 ideographic space, then the fixture word U+6211 and its count + final String lexicon = "\u3000\u6211 5000 r\n"; + final UnigramSegmenter loaded = UnigramSegmenter.load( + new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + Assertions.assertArrayEquals(new String[] {"\u6211"}, loaded.tokenize("\u6211")); + } } From 6716542d3eeeaffbf8873ca1782afcbe77a7b794 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:01:00 -0400 Subject: [PATCH 14/14] OPENNLP-1894: Address review: complete javadoc, hoist constants, and fold fixture duplication - Document the private lattice helpers decode, relax, and candidates, and the installer's boundedStream, with the parameter, return, and exception contracts the review expects every method to carry. - Document the WordEntry and Category record components and the double-array builder's findBase and ensureCapacity helpers. - Record on analyze, tokenize, and tokenizePos that a unk.def without a DEFAULT template leaves the lattice disconnected and makes them throw IllegalStateException. - State on readLines that it never returns an empty list, which is what lets the matrix.def header be read before the emptiness check. - Rename the Tokenizer override parameter from s to text in LatticeTokenizer and UnigramSegmenter, so the javadoc names a parameter that exists. - Hoist the matrix.def, char.def, and unk.def file names, the DEFAULT category name, the 0x code point prefix, the .. range separator, and the flag value into named constants in MecabDictionary, and let LatticeTokenizer reach the DEFAULT name through MecabDictionary instead of repeating the literal. - Name the tar block size, header field offsets, and field lengths in the TarGzArchives test helper instead of writing 512, 124, and 148 inline. - Replace the boolean[1] capture in candidates with a check that the candidate list is still empty, which is the same signal without the array. - Track the best boundary total in decode instead of recomputing the incumbent's connection cost on every comparison. - Drop the categories map field from MecabDictionary, which nothing read once the constructor resolved the DEFAULT category out of it. - Match the char.def code point prefix once, case insensitively, rather than testing 0x and 0X separately, and cut the range at the separator's own length. - Trim the matrix.def header before parsing it and report an empty first line as an empty matrix.def, since readLines never yields the empty list the previous check was looking for. - Split the omnibus malformed-dictionary test into named cases that pin the messages for a missing definition file, a char.def without DEFAULT, a lexicon with no entries, and an empty matrix.def. - Parameterize the malformed char.def cases and the malformed unigram lexicon cases, which were repeated assertThrows calls over one fixture shape. - Add a Morpheme test pinning the null and empty argument rejections and the defensive copy of the feature list. - Extend the invalid-argument tests to the entry points that were uncovered: MecabDictionary.load with a null directory or charset, the installer's null target, UnigramSegmenter's path and stream overloads, and both tokenize methods of each tokenizer. - Fold the repeated Files.write fixture calls into one write helper and hoist the shared lexicon, matrix, char.def, and unk.def fixture text into constants. - Correct dev/README-mecab-dictionaries.md to say that dicrc is the configuration file the distributions ship alongside the csv and def files a MecabDictionary reads, rather than implying the dictionary reads dicrc itself. --- dev/README-mecab-dictionaries.md | 2 +- .../tokenize/lattice/LatticeTokenizer.java | 65 ++++- .../tokenize/lattice/MecabDictionary.java | 140 ++++++--- .../lattice/MecabDictionaryInstaller.java | 10 +- .../tokenize/lattice/UnigramSegmenter.java | 32 +- .../lattice/LatticeTokenizerTest.java | 276 ++++++++++++------ .../lattice/MecabDictionaryInstallerTest.java | 2 + .../tools/tokenize/lattice/TarGzArchives.java | 48 ++- .../lattice/UnigramSegmenterTest.java | 32 +- 9 files changed, 424 insertions(+), 183 deletions(-) diff --git a/dev/README-mecab-dictionaries.md b/dev/README-mecab-dictionaries.md index fce653b397..6d1be9420f 100644 --- a/dev/README-mecab-dictionaries.md +++ b/dev/README-mecab-dictionaries.md @@ -40,7 +40,7 @@ On the first run without a checksum the script prints the SHA-256 it computed; r ## Step 2: unpack with the installer -The installer extracts only the files a `MecabDictionary` reads (`*.csv`, `*.def`, and `dicrc`), flattens them into the target directory, and by the same flattening makes it impossible for an archive path to escape that directory: +The installer extracts only the dictionary payload: the `*.csv` and `*.def` files a `MecabDictionary` reads, plus the `dicrc` configuration file the distributions ship alongside them. It flattens the entries into the target directory, and by the same flattening makes it impossible for an archive path to escape that directory: ```java import java.nio.file.Path; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index 7ce8be2aa4..649be0b80d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -95,6 +95,8 @@ private Node(int start, int end, WordEntry entry, boolean unknown) { * @return The morphemes in text order, spans in original coordinates, whitespace * omitted. Never {@code null}; empty for empty or all-whitespace input. * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position, which a {@code unk.def} without a {@code DEFAULT} template does. */ public List analyze(String text) { if (text == null) { @@ -122,11 +124,13 @@ public List analyze(String text) { * *

Reports the segmented surfaces, whitespace omitted.

* - * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position; see {@link #analyze(String)}. */ @Override - public String[] tokenize(String s) { - final List morphemes = analyze(s); + public String[] tokenize(String text) { + final List morphemes = analyze(text); final String[] tokens = new String[morphemes.size()]; for (int i = 0; i < tokens.length; i++) { tokens[i] = morphemes.get(i).surface(); @@ -139,11 +143,13 @@ public String[] tokenize(String s) { * *

Reports the segmented spans in original text coordinates, whitespace omitted.

* - * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position; see {@link #analyze(String)}. */ @Override - public Span[] tokenizePos(String s) { - final List morphemes = analyze(s); + public Span[] tokenizePos(String text) { + final List morphemes = analyze(text); final Span[] spans = new Span[morphemes.size()]; for (int i = 0; i < spans.length; i++) { spans[i] = morphemes.get(i).span(); @@ -151,7 +157,15 @@ public Span[] tokenizePos(String s) { return spans; } - /** Runs the Viterbi search over one whitespace-free stretch of text. */ + /** + * Runs the Viterbi search over one whitespace-free stretch of text. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param morphemes Receives the cheapest path's morphemes, in text order. + * @throws IllegalStateException Thrown if no path reaches the end of the stretch. + */ private void decode(String text, int from, int to, List morphemes) { final int length = to - from; // Each element heads the chain of nodes ending at that position. @@ -179,12 +193,13 @@ private void decode(String text, int from, int to, List morphemes) { } Node best = null; + long bestTotal = Long.MAX_VALUE; for (Node node = endingAt[length]; node != null; node = node.nextEndingHere) { final long total = node.pathCost + dictionary.connectionCost(node.entry.rightId(), BOUNDARY_CONTEXT); - if (best == null || total < best.pathCost - + dictionary.connectionCost(best.entry.rightId(), BOUNDARY_CONTEXT)) { + if (best == null || total < bestTotal) { best = node; + bestTotal = total; } } if (best == null) { @@ -202,7 +217,13 @@ private void decode(String text, int from, int to, List morphemes) { } } - /** Connects a candidate to the cheapest predecessor ending where it starts. */ + /** + * Connects a candidate to the cheapest predecessor ending where it starts. + * + * @param candidate The node to give a path cost and a predecessor. + * @param predecessors The head of the chain of nodes ending where the candidate + * starts, or {@code null} when it starts at the stretch start. + */ private void relax(Node candidate, Node predecessors) { if (predecessors == null) { candidate.pathCost = candidate.entry.cost() @@ -251,18 +272,31 @@ private void computeCategoryRuns(String text, int from, int to, } } - /** Gathers lexicon matches and unknown-word candidates starting at one position. */ + /** + * Gathers lexicon matches and unknown-word candidates starting at one position. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end, which no candidate may reach past. + * @param offset The candidate start, relative to {@code from}. + * @param positionCategory The category of that position, or {@code null} for a + * position inside a surrogate pair. + * @param positionRunEnd The exclusive end of the same-category run starting there, + * meaningful only when {@code positionCategory} is not + * {@code null}. + * @param candidates Receives the candidates. Must be empty on entry. + * @throws IllegalStateException Thrown if neither the lexicon, the position's + * category, nor the {@code DEFAULT} template offers a candidate. + */ private void candidates(String text, int from, int to, int offset, Category positionCategory, int positionRunEnd, List candidates) { final int position = from + offset; - final boolean[] matched = new boolean[1]; dictionary.prefixMatches(text, position, to, (length, entries) -> { - matched[0] = true; for (final WordEntry entry : entries) { candidates.add(new Node(position, position + length, entry, false)); } }); - final boolean lexiconMatch = matched[0]; + final boolean lexiconMatch = !candidates.isEmpty(); final int codePoint = text.codePointAt(position); final Category category; @@ -286,7 +320,8 @@ private void candidates(String text, int from, int to, int offset, if (candidates.isEmpty()) { // Neither the lexicon nor the character's category produced a candidate here, so a // single-character entry from the DEFAULT template keeps the lattice connected. - final List fallback = dictionary.unknownEntries("DEFAULT"); + final List fallback = + dictionary.unknownEntries(MecabDictionary.DEFAULT_CATEGORY); if (fallback != null) { for (final WordEntry entry : fallback) { candidates.add( diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index a971c586a3..59d9ab3388 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -49,11 +49,46 @@ */ public final class MecabDictionary { - /** One lexicon or unknown-word entry. */ + /** + * The category name every {@code char.def} must define; unmapped code points and + * unknown-word handling fall back to it. + */ + static final String DEFAULT_CATEGORY = "DEFAULT"; + + private static final String MATRIX_DEF = "matrix.def"; + private static final String CHAR_DEF = "char.def"; + private static final String UNK_DEF = "unk.def"; + + /** The prefix a {@code char.def} code point field carries, in either letter case. */ + private static final String HEX_PREFIX = "0x"; + + /** The separator between the two ends of a {@code char.def} code point range. */ + private static final String RANGE_SEPARATOR = ".."; + + /** The {@code char.def} field value that turns a category flag on. */ + private static final String FLAG_ON = "1"; + + /** + * One lexicon or unknown-word entry. + * + * @param leftId The left context id, an index into the connection matrix. + * @param rightId The right context id, an index into the connection matrix. + * @param cost The entry's own cost. + * @param features The entry's feature columns, in file order. + */ record WordEntry(int leftId, int rightId, int cost, List features) { } - /** One character category's unknown-word behavior from {@code char.def}. */ + /** + * One character category's unknown-word behavior from {@code char.def}. + * + * @param name The category name. + * @param invoke Whether unknown-word candidates are generated even where the lexicon + * matched. + * @param group Whether a whole run of same-category characters is offered as one + * candidate. + * @param length How many leading characters of the run are offered as candidates. + */ record Category(String name, boolean invoke, boolean group, int length) { } @@ -236,6 +271,10 @@ private void insert(int left, int right, int depth, int state) { * Finds the lowest base at which every label lands on a free slot. Labels * arrive in surface-character order, not numeric order, so the smallest and * largest label are computed rather than assumed positional. + * + * @param labels The child labels to place. + * @param labelCount How many leading elements of {@code labels} are in use. + * @return The base offset every label fits at. */ private int findBase(int[] labels, int labelCount) { int smallest = labels[0]; @@ -261,6 +300,11 @@ private int findBase(int[] labels, int labelCount) { } } + /** + * Grows the base and check arrays until a slot is addressable. + * + * @param slot The highest slot index that has to be writable. + */ private void ensureCapacity(int slot) { if (slot >= check.length) { int capacity = check.length; @@ -358,7 +402,10 @@ private void map(int from, int to, String category) { /** * Folds the recorded mappings into their lookup table. * + * @param categories The categories the {@code char.def} category section defined, + * keyed by name. * @return The table. Never {@code null}. + * @throws IOException Thrown if a mapping names a category that was never defined. */ private CategoryTable build(Map categories) throws IOException { // Cut the supplementary ranges at every boundary they introduce, so that each @@ -412,13 +459,19 @@ private CategoryTable build(Map categories) throws IOException * Resolves a mapped category name against the defined categories, so a mapping to * a name the {@code char.def} category section never defined fails at load with * the offending code point instead of falling back silently at lookup time. + * + * @param name The category name a mapping line gave. + * @param categories The defined categories, keyed by name. + * @param codePoint A code point the mapping covers, for the error message. + * @return The resolved category. Never {@code null}. + * @throws IOException Thrown if no category of that name was defined. */ - private static Category resolve(String name, Map categories, + private Category resolve(String name, Map categories, int codePoint) throws IOException { final Category category = categories.get(name); if (category == null) { throw new IOException(String.format( - "char.def maps U+%04X to the undefined category %s", codePoint, name)); + CHAR_DEF + " maps U+%04X to the undefined category %s", codePoint, name)); } return category; } @@ -455,7 +508,6 @@ interface PrefixMatchConsumer { private final DoubleArrayLexicon lexicon; private final short[] connectionCosts; private final int rightSize; - private final Map categories; private final CategoryTable categoryTable; private final Category defaultCategory; private final Map> unknownEntries; @@ -466,9 +518,8 @@ private MecabDictionary(DoubleArrayLexicon lexicon, this.lexicon = lexicon; this.connectionCosts = connectionCosts; this.rightSize = rightSize; - this.categories = categories; this.categoryTable = categoryTable; - this.defaultCategory = categories.get("DEFAULT"); + this.defaultCategory = categories.get(DEFAULT_CATEGORY); this.unknownEntries = unknownEntries; } @@ -507,23 +558,24 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } // The connection matrix is read first because its dimensions are what every // lexicon entry's context ids have to be inside of. - final List matrixLines = readLines(directory.resolve("matrix.def"), charset); - if (matrixLines.isEmpty()) { - throw new IOException("empty matrix.def under " + directory); + final List matrixLines = readLines(directory.resolve(MATRIX_DEF), charset); + final String headerLine = StringUtil.trimUnicodeWhitespace(matrixLines.get(0)); + if (headerLine.isEmpty()) { + throw new IOException("empty " + MATRIX_DEF + " under " + directory); } - final String[] header = splitWhitespace(matrixLines.get(0)); + final String[] header = splitWhitespace(headerLine); if (header.length != 2) { - throw new IOException("malformed matrix.def header: " + matrixLines.get(0)); + throw new IOException("malformed " + MATRIX_DEF + " header: " + headerLine); } - final int leftSize = parseInt(header[0], "matrix.def", 1); - final int rightSize = parseInt(header[1], "matrix.def", 1); + final int leftSize = parseInt(header[0], MATRIX_DEF, 1); + final int rightSize = parseInt(header[1], MATRIX_DEF, 1); if (leftSize < 1 || rightSize < 1) { - throw new IOException("matrix.def dimensions must be positive, got " + throw new IOException(MATRIX_DEF + " dimensions must be positive, got " + leftSize + " " + rightSize); } final long cells = (long) leftSize * rightSize; if (cells > Integer.MAX_VALUE) { - throw new IOException("matrix.def dimensions " + leftSize + " x " + rightSize + throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize + " overflow the addressable connection matrix"); } final short[] costs = new short[(int) cells]; @@ -534,18 +586,18 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } final String[] fields = splitWhitespace(line); if (fields.length != 3) { - throw new IOException("malformed matrix.def line " + (i + 1)); + throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1)); } - final int right = parseInt(fields[0], "matrix.def", i + 1); - final int left = parseInt(fields[1], "matrix.def", i + 1); + final int right = parseInt(fields[0], MATRIX_DEF, i + 1); + final int left = parseInt(fields[1], MATRIX_DEF, i + 1); if (right < 0 || right >= leftSize || left < 0 || left >= rightSize) { - throw new IOException("malformed matrix.def line " + (i + 1) + throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1) + ": context ids " + right + " " + left + " are outside the declared dimensions " + leftSize + " " + rightSize); } - final int cost = parseInt(fields[2], "matrix.def", i + 1); + final int cost = parseInt(fields[2], MATRIX_DEF, i + 1); if (cost < Short.MIN_VALUE || cost > Short.MAX_VALUE) { - throw new IOException("malformed matrix.def line " + (i + 1) + throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1) + ": connection cost " + cost + " is outside the 16-bit range the" + " format defines"); } @@ -564,10 +616,10 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc final Map categories = new HashMap<>(); final CategoryTableBuilder categoryTable = new CategoryTableBuilder(); - readCharacterDefinition(directory.resolve("char.def"), charset, categories, + readCharacterDefinition(directory.resolve(CHAR_DEF), charset, categories, categoryTable); final Map> unknown = new HashMap<>(); - readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); + readLexicon(directory.resolve(UNK_DEF), charset, unknown, leftSize, rightSize); return new MecabDictionary(DoubleArrayLexicon.build(lexicon), costs, rightSize, categories, categoryTable.build(categories), unknown); @@ -608,12 +660,12 @@ private static void readLexicon(Path file, Charset charset, final int rightId = parseInt(fields.get(2), file.toString(), lineNumber); if (leftId < 0 || leftId >= rightSize) { throw new IOException("malformed entry at " + file + " line " + lineNumber - + ": left context id " + leftId + " is outside the matrix.def dimensions " + + ": left context id " + leftId + " is outside the " + MATRIX_DEF + " dimensions " + leftSize + " " + rightSize); } if (rightId < 0 || rightId >= leftSize) { throw new IOException("malformed entry at " + file + " line " + lineNumber - + ": right context id " + rightId + " is outside the matrix.def dimensions " + + ": right context id " + rightId + " is outside the " + MATRIX_DEF + " dimensions " + leftSize + " " + rightSize); } final WordEntry entry = new WordEntry(leftId, rightId, @@ -623,7 +675,18 @@ private static void readLexicon(Path file, Charset charset, } } - /** Reads char.def: category behavior lines and code point mapping lines. */ + /** + * Reads {@code char.def}: the category behavior lines and the code point mapping + * lines, in file order, so that a later mapping wins over an earlier one. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @param categories Receives the defined categories, keyed by name. + * @param categoryTable Receives the code point to category name mappings. + * @throws IOException Thrown if the file is missing, a line is malformed, a code + * point is outside the Unicode range, a range descends, or the file defines + * no {@code DEFAULT} category. + */ private static void readCharacterDefinition(Path file, Charset charset, Map categories, CategoryTableBuilder categoryTable) throws IOException { @@ -635,13 +698,14 @@ private static void readCharacterDefinition(Path file, Charset charset, continue; } final String[] fields = splitWhitespace(line); - if (fields[0].startsWith("0x") || fields[0].startsWith("0X")) { - final int rangeSeparator = fields[0].indexOf(".."); + if (fields[0].regionMatches(true, 0, HEX_PREFIX, 0, HEX_PREFIX.length())) { + final int rangeSeparator = fields[0].indexOf(RANGE_SEPARATOR); final int from; final int to; if (rangeSeparator >= 0) { from = parseCodePoint(fields[0].substring(0, rangeSeparator), file, lineNumber); - to = parseCodePoint(fields[0].substring(rangeSeparator + 2), file, lineNumber); + to = parseCodePoint( + fields[0].substring(rangeSeparator + RANGE_SEPARATOR.length()), file, lineNumber); } else { from = parseCodePoint(fields[0], file, lineNumber); to = from; @@ -659,12 +723,12 @@ private static void readCharacterDefinition(Path file, Charset charset, throw new IOException("malformed category at " + file + " line " + lineNumber); } categories.put(fields[0], new Category(fields[0], - "1".equals(fields[1]), "1".equals(fields[2]), + FLAG_ON.equals(fields[1]), FLAG_ON.equals(fields[2]), parseInt(fields[3], file.toString(), lineNumber))); } } - if (!categories.containsKey("DEFAULT")) { - throw new IOException("char.def defines no DEFAULT category: " + file); + if (!categories.containsKey(DEFAULT_CATEGORY)) { + throw new IOException(CHAR_DEF + " defines no " + DEFAULT_CATEGORY + " category: " + file); } } @@ -721,7 +785,8 @@ List unknownEntries(String category) { * * @param file The file to read. * @param charset The encoding to decode with. - * @return The lines without their line terminators. Never {@code null}. + * @return The lines without their line terminators. Never {@code null} and never + * empty: an empty file reads as one empty line. * @throws IOException Thrown if the file is missing or reading fails. */ private static List readLines(Path file, Charset charset) throws IOException { @@ -822,14 +887,15 @@ private static int parseInt(String text, String file, int lineNumber) * @param file The file being read, for the error message. * @param lineNumber The line being read, for the error message. * @return The parsed code point, which may be in a supplementary plane. - * @throws IOException Thrown if the field is not a valid hexadecimal code point or - * names a value no Unicode code point has. + * @throws IOException Thrown if the field is shorter than the prefix, is not a valid + * hexadecimal number, or names a value no Unicode code point has. */ private static int parseCodePoint(String text, Path file, int lineNumber) throws IOException { final int codePoint; try { - codePoint = Integer.parseInt(StringUtil.trimUnicodeWhitespace(text).substring(2), 16); + codePoint = Integer.parseInt( + StringUtil.trimUnicodeWhitespace(text).substring(HEX_PREFIX.length()), 16); } catch (RuntimeException e) { throw new IOException("malformed code point in " + file + " line " + lineNumber, e); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 3c2589bc02..76212b1b5d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -246,7 +246,15 @@ private static void skip(InputStream in, long bytes) throws IOException { } } - /** Wraps the tar stream so exactly one entry's bytes are readable. */ + /** + * Wraps the tar stream so exactly one entry's bytes are readable. + * + * @param in The tar stream, positioned at the entry's first byte. + * @param size The entry size in bytes. + * @return A stream reporting end of stream after that many bytes, and failing if the + * tar stream ends first. Never {@code null}; closing it leaves {@code in} + * open and positioned after the entry content. + */ private static InputStream boundedStream(InputStream in, long size) { return new InputStream() { private long remaining = size; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index 00469ac4f7..060a0a53d2 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -183,14 +183,14 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) * *

Reports the segmented surfaces, whitespace omitted.

* - * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. */ @Override - public String[] tokenize(String s) { - final Span[] spans = tokenizePos(s); + public String[] tokenize(String text) { + final Span[] spans = tokenizePos(text); final String[] tokens = new String[spans.length]; for (int i = 0; i < tokens.length; i++) { - tokens[i] = s.substring(spans[i].getStart(), spans[i].getEnd()); + tokens[i] = text.substring(spans[i].getStart(), spans[i].getEnd()); } return tokens; } @@ -200,31 +200,39 @@ public String[] tokenize(String s) { * *

Reports the segmented spans in original text coordinates, whitespace omitted.

* - * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. */ @Override - public Span[] tokenizePos(String s) { - if (s == null) { + public Span[] tokenizePos(String text) { + if (text == null) { throw new IllegalArgumentException("text must not be null"); } final List spans = new ArrayList<>(); int start = 0; - while (start < s.length()) { - if (StringUtil.isWhitespace(s.charAt(start))) { + while (start < text.length()) { + if (StringUtil.isWhitespace(text.charAt(start))) { start++; continue; } int end = start; - while (end < s.length() && !StringUtil.isWhitespace(s.charAt(end))) { + while (end < text.length() && !StringUtil.isWhitespace(text.charAt(end))) { end++; } - decode(s, start, end, spans); + decode(text, start, end, spans); start = end; } return spans.toArray(new Span[0]); } - /** Viterbi over word log-probabilities within one whitespace-free stretch. */ + /** + * Viterbi over word log-probabilities within one whitespace-free stretch. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param spans Receives the best path's spans, in text order and in original text + * coordinates. + */ private void decode(String text, int from, int to, List spans) { final int length = to - from; final double[] best = new double[length + 1]; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 034cd6d8c3..56a3ea8710 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -21,12 +21,15 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.util.Span; @@ -40,6 +43,24 @@ */ public class LatticeTokenizerTest { + private static final String LEXICON_CSV = "lexicon.csv"; + private static final String MATRIX_DEF = "matrix.def"; + private static final String CHAR_DEF = "char.def"; + private static final String UNK_DEF = "unk.def"; + + /** A one by one connection matrix charging cost zero, for single-context fixtures. */ + private static final String UNIT_MATRIX = "1 1\n0 0 0\n"; + + /** + * The {@code char.def} line defining the DEFAULT category: it does not invoke + * unknown-word handling beside a lexicon match, it groups a whole run into one + * candidate, and it offers no fixed-length candidates. + */ + private static final String DEFAULT_CATEGORY_LINE = "DEFAULT 0 1 0"; + + /** The {@code unk.def} template line for the DEFAULT category. */ + private static final String DEFAULT_UNKNOWN_TEMPLATE = "DEFAULT,0,0,10000,symbol,unknown"; + @TempDir static Path directory; @@ -47,7 +68,7 @@ public class LatticeTokenizerTest { @BeforeAll static void loadDictionary() throws IOException { - write("lexicon.csv", String.join("\n", + write(LEXICON_CSV, String.join("\n", "\u6771\u4EAC,0,0,3000,noun,proper", "\u4EAC\u90FD,0,0,3000,noun,proper", "\u6771,0,0,6000,noun,common", @@ -55,9 +76,9 @@ static void loadDictionary() throws IOException { "\u306B,0,0,1000,particle,case", "\u884C\u304F,0,0,3000,verb,base", "")); - write("matrix.def", "1 1\n0 0 0\n"); - write("char.def", String.join("\n", - "DEFAULT 0 1 0", + write(MATRIX_DEF, UNIT_MATRIX); + write(CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, "KANJI 0 0 2", "HIRAGANA 0 1 0", "LATIN 1 1 0", @@ -67,8 +88,8 @@ static void loadDictionary() throws IOException { "0x0041..0x005A LATIN", "0x0061..0x007A LATIN", "")); - write("unk.def", String.join("\n", - "DEFAULT,0,0,10000,symbol,unknown", + write(UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, "LATIN,0,0,4000,noun,foreign", "KANJI,0,0,8000,noun,unknown", "HIRAGANA,0,0,9000,particle,unknown", @@ -76,13 +97,19 @@ static void loadDictionary() throws IOException { tokenizer = new LatticeTokenizer(MecabDictionary.load(directory)); } + /** Writes one UTF-8 dictionary file into the shared dictionary directory. */ private static void write(String name, String content) throws IOException { - Files.write(directory.resolve(name), content.getBytes(StandardCharsets.UTF_8)); + write(directory, name, content); + } + + /** Writes one UTF-8 dictionary file into a test-supplied directory. */ + private static void write(Path target, String name, String content) throws IOException { + Files.write(target.resolve(name), content.getBytes(StandardCharsets.UTF_8)); } @Test void testLatticePrefersTheCheaperSegmentation() { - // the famous case: Tokyo+Metro must win over East+Kyoto + // Tokyo plus the metropolis suffix must beat the competing reading east plus Kyoto. final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, @@ -249,8 +276,7 @@ void testSpansStayOriginalAfterLeadingWhitespace() { void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { // The rest of the dictionary is well formed, so the short row is what load rejects. writeUnitMatrixDictionary(broken); - Files.write(broken.resolve("lexicon.csv"), - "\u6771,0,0\n".getBytes(StandardCharsets.UTF_8)); + write(broken, LEXICON_CSV, "\u6771,0,0\n"); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -262,8 +288,7 @@ void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { // The rest of the dictionary is well formed, so the cost column is what load rejects. writeUnitMatrixDictionary(broken); - Files.write(broken.resolve("lexicon.csv"), - "\u6771,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); + write(broken, LEXICON_CSV, "\u6771,0,0,abc,noun\n"); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -273,9 +298,8 @@ void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException */ @Test void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { - Files.write(broken.resolve("lexicon.csv"), - "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(broken.resolve("matrix.def"), "1 1\n0 0\n".getBytes(StandardCharsets.UTF_8)); + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, "1 1\n0 0\n"); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -286,11 +310,9 @@ void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { @Test void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) throws IOException { - Files.write(broken.resolve("lexicon.csv"), - "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(broken.resolve("char.def"), - "DEFAULT 0 1 0\n0x4E00..0x9FFF\n".getBytes(StandardCharsets.UTF_8)); + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, UNIT_MATRIX); + write(broken, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\n0x4E00..0x9FFF\n"); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -303,30 +325,90 @@ void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) @Test void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) throws IOException { - Files.write(partial.resolve("lexicon.csv"), - "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(partial.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(partial.resolve("char.def"), - "DEFAULT 0 1 0\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n" - .getBytes(StandardCharsets.UTF_8)); - Files.write(partial.resolve("unk.def"), - "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); + write(partial, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(partial, MATRIX_DEF, UNIT_MATRIX); + write(partial, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n"); + write(partial, UNK_DEF, "KANJI,0,0,8000,noun\n"); final LatticeTokenizer limited = new LatticeTokenizer(MecabDictionary.load(partial)); Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("\u2460")); } + /** + * Verifies that a directory holding a lexicon but none of the definition files is + * rejected at load time, naming the first file that is missing. + */ @Test - void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException { - Files.write(broken.resolve("lexicon.csv"), - "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); - Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + void testMissingDefinitionFileFailsLoud(@TempDir Path broken) throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("required dictionary file is missing: " + + broken.resolve(MATRIX_DEF), e.getMessage()); + } + + /** + * Verifies that a {@code char.def} without the mandatory DEFAULT category is rejected + * at load time rather than leaving unmapped code points without a fallback. + */ + @Test + void testCharDefWithoutDefaultCategoryFailsLoud(@TempDir Path broken) throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, UNIT_MATRIX); + write(broken, CHAR_DEF, "KANJI 0 0 2\n0x4E00..0x9FFF KANJI\n"); + write(broken, UNK_DEF, "KANJI,0,0,8000,noun\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("char.def defines no DEFAULT category: " + + broken.resolve(CHAR_DEF), e.getMessage()); + } + + /** + * Verifies that a directory with the definition files but no lexicon entry at all is + * rejected at load time, since no text could be segmented against it. + */ + @Test + void testDictionaryWithoutLexiconEntriesFailsLoud(@TempDir Path empty) throws IOException { + writeUnitMatrixDictionary(empty); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(empty)); + Assertions.assertEquals("no lexicon entries found under " + empty, e.getMessage()); + } + + /** + * Verifies that an empty {@code matrix.def} is reported as such instead of as a + * malformed header with nothing to show. + */ + @Test + void testEmptyMatrixDefFailsLoud(@TempDir Path broken) throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, ""); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("empty matrix.def under " + broken, e.getMessage()); + } - Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(broken.resolve("char.def"), - "KANJI 0 0 2\n0x4E00..0x9FFF KANJI\n".getBytes(StandardCharsets.UTF_8)); - Files.write(broken.resolve("unk.def"), - "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); + /** + * Verifies the {@code char.def} fail-loud paths that a malformed line can take: a + * descending code point range, a code point outside the Unicode range, a code point + * field that is not hexadecimal, and a category line missing its length column. + * + * @param charDef The {@code char.def} content under test. + * @param broken The directory the fixture dictionary is written into. + * @throws IOException Thrown if writing the fixture fails. + */ + @ParameterizedTest(name = "[{index}] char.def {0}") + @ValueSource(strings = { + DEFAULT_CATEGORY_LINE + "\n0x0110..0x0100 LATIN\n", + DEFAULT_CATEGORY_LINE + "\n0x110000 LATIN\n", + DEFAULT_CATEGORY_LINE + "\n0xZZ LATIN\n", + "DEFAULT 0 1\n"}) + void testMalformedCharDefFailsLoud(String charDef, @TempDir Path broken) + throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, UNIT_MATRIX); + write(broken, CHAR_DEF, charDef); + write(broken, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -339,23 +421,22 @@ void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException * @throws IOException Thrown if writing any of the files fails. */ private static void writeSupplementaryDictionary(Path target) throws IOException { - Files.write(target.resolve("lexicon.csv"), - "\u6771,0,0,6000,noun,common\n".getBytes(StandardCharsets.UTF_8)); - Files.write(target.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(target.resolve("char.def"), String.join("\n", - "DEFAULT 0 1 0", + write(target, LEXICON_CSV, "\u6771,0,0,6000,noun,common\n"); + write(target, MATRIX_DEF, UNIT_MATRIX); + write(target, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, "KANJI 0 0 2", "LATIN 1 1 0", "", "0x4E00..0x9FFF KANJI", "0x20000..0x2A6DF KANJI", "0x0061..0x007A LATIN", - "").getBytes(StandardCharsets.UTF_8)); - Files.write(target.resolve("unk.def"), String.join("\n", - "DEFAULT,0,0,10000,symbol,unknown", + "")); + write(target, UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, "KANJI,0,0,8000,noun,unknown", "LATIN,0,0,4000,noun,foreign", - "").getBytes(StandardCharsets.UTF_8)); + "")); } /** @@ -432,11 +513,9 @@ void testSupplementaryIdeographDoesNotAbsorbItsNeighbour(@TempDir Path supplemen * @throws IOException Thrown if writing any of the files fails. */ private static void writeUnitMatrixDictionary(Path target) throws IOException { - Files.write(target.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(target.resolve("char.def"), - "DEFAULT 0 1 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(target.resolve("unk.def"), - "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + write(target, MATRIX_DEF, UNIT_MATRIX); + write(target, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\n"); + write(target, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); } /** @@ -449,11 +528,10 @@ private static void writeUnitMatrixDictionary(Path target) throws IOException { void testRightContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) throws IOException { writeUnitMatrixDictionary(mismatched); - Files.write(mismatched.resolve("lexicon.csv"), - "\u6771,0,5,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + write(mismatched, LEXICON_CSV, "\u6771,0,5,3000,noun\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(mismatched)); - Assertions.assertEquals("malformed entry at " + mismatched.resolve("lexicon.csv") + Assertions.assertEquals("malformed entry at " + mismatched.resolve(LEXICON_CSV) + " line 1: right context id 5 is outside the matrix.def dimensions 1 1", e.getMessage()); } @@ -467,11 +545,10 @@ void testRightContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) void testLeftContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) throws IOException { writeUnitMatrixDictionary(mismatched); - Files.write(mismatched.resolve("lexicon.csv"), - "\u6771,0,0,3000,noun\n\u90FD,7,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + write(mismatched, LEXICON_CSV, "\u6771,0,0,3000,noun\n\u90FD,7,0,3000,noun\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(mismatched)); - Assertions.assertEquals("malformed entry at " + mismatched.resolve("lexicon.csv") + Assertions.assertEquals("malformed entry at " + mismatched.resolve(LEXICON_CSV) + " line 2: left context id 7 is outside the matrix.def dimensions 1 1", e.getMessage()); } @@ -482,7 +559,38 @@ void testInvalidArguments() { () -> new LatticeTokenizer(null)); Assertions.assertThrows(IllegalArgumentException.class, () -> MecabDictionary.load(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionary.load(null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionary.load(directory, null)); Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.analyze(null)); + Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.tokenize(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> tokenizer.tokenizePos(null)); + } + + /** + * Verifies the {@link Morpheme} contract every segmentation result is built from: a + * {@code null} span, a {@code null} or empty surface, and {@code null} features are + * all rejected, and the feature list is copied so a later change to the caller's list + * cannot be seen through the morpheme. + */ + @Test + void testMorphemeRejectsInvalidArguments() { + final Span span = new Span(0, 1); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(null, "東", List.of("noun"), false)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(span, null, List.of("noun"), false)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(span, "", List.of("noun"), false)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(span, "東", null, false)); + + final List features = new ArrayList<>(List.of("noun")); + final Morpheme morpheme = new Morpheme(span, "東", features, false); + features.add("proper"); + Assertions.assertEquals(List.of("noun"), morpheme.features()); } /** @@ -494,20 +602,17 @@ void testInvalidArguments() { @Test void testLaterSupplementaryMappingWinsInsideAnEarlierRange(@TempDir Path overlapped) throws IOException { - Files.write(overlapped.resolve("lexicon.csv"), - "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(overlapped.resolve("matrix.def"), - "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(overlapped.resolve("char.def"), String.join("\n", - "DEFAULT 0 1 0", + write(overlapped, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(overlapped, MATRIX_DEF, UNIT_MATRIX); + write(overlapped, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, "KANJI 0 0 2", "LATIN 1 1 0", "", "0x20000..0x2FFFF KANJI", "0x24000..0x25000 LATIN", - "").getBytes(StandardCharsets.UTF_8)); - Files.write(overlapped.resolve("unk.def"), - "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + "")); + write(overlapped, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); final MecabDictionary dictionary = MecabDictionary.load(overlapped); Assertions.assertEquals("KANJI", dictionary.categoryOf(0x20000).name()); @@ -527,18 +632,15 @@ void testLaterSupplementaryMappingWinsInsideAnEarlierRange(@TempDir Path overlap @Test void testCharDefRangeStraddlingTheBmpBoundary(@TempDir Path straddling) throws IOException { - Files.write(straddling.resolve("lexicon.csv"), - "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(straddling.resolve("matrix.def"), - "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(straddling.resolve("char.def"), String.join("\n", - "DEFAULT 0 1 0", + write(straddling, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(straddling, MATRIX_DEF, UNIT_MATRIX); + write(straddling, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, "LATIN 1 1 0", "", "0xFF00..0x10040 LATIN", - "").getBytes(StandardCharsets.UTF_8)); - Files.write(straddling.resolve("unk.def"), - "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + "")); + write(straddling, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); final MecabDictionary dictionary = MecabDictionary.load(straddling); Assertions.assertEquals("LATIN", dictionary.categoryOf(0xFF00).name()); @@ -556,17 +658,14 @@ void testCharDefRangeStraddlingTheBmpBoundary(@TempDir Path straddling) */ @Test void testMappingToUndefinedCategoryFailsLoud(@TempDir Path ghost) throws IOException { - Files.write(ghost.resolve("lexicon.csv"), - "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(ghost.resolve("matrix.def"), - "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(ghost.resolve("char.def"), String.join("\n", - "DEFAULT 0 1 0", + write(ghost, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(ghost, MATRIX_DEF, UNIT_MATRIX); + write(ghost, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, "", "0x0100..0x0110 GHOST", - "").getBytes(StandardCharsets.UTF_8)); - Files.write(ghost.resolve("unk.def"), - "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + "")); + write(ghost, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(ghost)); @@ -582,8 +681,7 @@ void testMappingToUndefinedCategoryFailsLoud(@TempDir Path ghost) throws IOExcep @Test void testMatrixCostOutsideShortRangeFailsLoud(@TempDir Path broken) throws IOException { writeUnitMatrixDictionary(broken); - Files.write(broken.resolve("matrix.def"), - "1 1\n0 0 40000\n".getBytes(StandardCharsets.UTF_8)); + write(broken, MATRIX_DEF, "1 1\n0 0 40000\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); Assertions.assertEquals("malformed matrix.def line 2: connection cost 40000 is" @@ -599,8 +697,7 @@ void testMatrixCostOutsideShortRangeFailsLoud(@TempDir Path broken) throws IOExc void testMatrixDimensionProductBeyondIntRangeFailsLoud(@TempDir Path broken) throws IOException { writeUnitMatrixDictionary(broken); - Files.write(broken.resolve("matrix.def"), - "70000 70000\n".getBytes(StandardCharsets.UTF_8)); + write(broken, MATRIX_DEF, "70000 70000\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); Assertions.assertEquals("matrix.def dimensions 70000 x 70000 overflow the" @@ -615,8 +712,7 @@ void testMatrixDimensionProductBeyondIntRangeFailsLoud(@TempDir Path broken) void testMatrixRowContextIdsOutsideDimensionsFailLoud(@TempDir Path broken) throws IOException { writeUnitMatrixDictionary(broken); - Files.write(broken.resolve("matrix.def"), - "1 1\n2 0 5\n".getBytes(StandardCharsets.UTF_8)); + write(broken, MATRIX_DEF, "1 1\n2 0 5\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); Assertions.assertEquals("malformed matrix.def line 2: context ids 2 0 are outside" diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java index da95c97366..fb5c6a701f 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -85,6 +85,8 @@ void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target) void testInvalidArguments(@TempDir Path target) { Assertions.assertThrows(IllegalArgumentException.class, () -> MecabDictionaryInstaller.install(null, target)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.install(target.toUri(), null)); Assertions.assertThrows(IllegalArgumentException.class, () -> MecabDictionaryInstaller.extract(null, target)); Assertions.assertThrows(IllegalArgumentException.class, diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java index 4fdfb29d01..896115aba2 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java @@ -28,6 +28,21 @@ */ final class TarGzArchives { + /** The tar block size; headers, content, and padding are all whole blocks. */ + private static final int BLOCK = 512; + + /** The header field offsets and lengths this builder writes, in bytes. */ + private static final int NAME_LENGTH = 100; + private static final int MODE_OFFSET = 100; + private static final int SIZE_OFFSET = 124; + private static final int SIZE_LENGTH = 12; + private static final int CHECKSUM_OFFSET = 148; + private static final int CHECKSUM_LENGTH = 8; + private static final int TYPE_OFFSET = 156; + + /** The type flag of a regular file entry. */ + private static final char REGULAR_FILE = '0'; + private TarGzArchives() { } @@ -45,7 +60,8 @@ static byte[] gzippedTar(String[][] entries) throws IOException { for (final String[] entry : entries) { tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); } - tar.write(new byte[1024]); + // Two zero blocks end a tar archive. + tar.write(new byte[2 * BLOCK]); final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { gzip.write(tar.toByteArray()); @@ -67,31 +83,35 @@ static byte[] gzippedTar(String[][] entries) throws IOException { private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) throws IOException { final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); - if (nameBytes.length == 0 || nameBytes.length > 100) { + if (nameBytes.length == 0 || nameBytes.length > NAME_LENGTH) { throw new IllegalArgumentException( - "entry name must be 1..100 bytes, got " + nameBytes.length); + "entry name must be 1.." + NAME_LENGTH + " bytes, got " + nameBytes.length); } - final byte[] header = new byte[512]; + final byte[] header = new byte[BLOCK]; System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); - System.arraycopy(mode, 0, header, 100, mode.length); - final String size = String.format("%011o", content.length); - System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); - header[156] = '0'; - for (int i = 148; i < 156; i++) { + System.arraycopy(mode, 0, header, MODE_OFFSET, mode.length); + // Both numeric fields hold octal digits followed by one terminator byte. + final byte[] size = String.format("%0" + (SIZE_LENGTH - 1) + "o", content.length) + .getBytes(StandardCharsets.US_ASCII); + System.arraycopy(size, 0, header, SIZE_OFFSET, size.length); + header[TYPE_OFFSET] = REGULAR_FILE; + // The checksum is computed with its own field read as spaces. + for (int i = CHECKSUM_OFFSET; i < CHECKSUM_OFFSET + CHECKSUM_LENGTH; i++) { header[i] = ' '; } int checksum = 0; for (final byte b : header) { checksum += b & 0xFF; } - final String checksumText = String.format("%06o", checksum); - System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); - header[154] = 0; - header[155] = ' '; + final byte[] checksumText = String.format("%0" + (CHECKSUM_LENGTH - 2) + "o", checksum) + .getBytes(StandardCharsets.US_ASCII); + System.arraycopy(checksumText, 0, header, CHECKSUM_OFFSET, checksumText.length); + header[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 2] = 0; + header[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 1] = ' '; tar.write(header); tar.write(content); - final int padding = (512 - content.length % 512) % 512; + final int padding = (BLOCK - content.length % BLOCK) % BLOCK; tar.write(new byte[padding]); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index 977d42fcc4..487c1c6ad3 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -19,11 +19,15 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.util.Span; @@ -153,26 +157,28 @@ void testSpansStayOriginalAfterLeadingWhitespace() { segmenter.tokenizePos(text)); } - @Test - void testMalformedLexiconsFailLoud() { - Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( - new ByteArrayInputStream("word\n".getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8)); - Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( - new ByteArrayInputStream("word abc\n".getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8)); - Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( - new ByteArrayInputStream("word 0\n".getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8)); + @ParameterizedTest(name = "lexicon content \"{0}\"") + @ValueSource(strings = {"word\n", "word abc\n", "word 0\n", "\n\n"}) + void testMalformedLexiconsFailLoud(String lexicon) { Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( - new ByteArrayInputStream("\n\n".getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); } @Test void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class, - () -> UnigramSegmenter.load((java.nio.file.Path) null)); + () -> UnigramSegmenter.load((Path) null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((Path) null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load(Path.of("words.txt"), null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((InputStream) null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load(new ByteArrayInputStream(new byte[0]), null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> segmenter.tokenize(null)); Assertions.assertThrows(IllegalArgumentException.class, () -> segmenter.tokenizePos(null)); }