diff --git a/dev/README-mecab-dictionaries.md b/dev/README-mecab-dictionaries.md new file mode 100644 index 0000000000..6d1be9420f --- /dev/null +++ b/dev/README-mecab-dictionaries.md @@ -0,0 +1,85 @@ + + +# CJK dictionaries for the lattice tokenizer + +The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a mecab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and each dictionary carries its own license. Read the license file inside the archive before use. + +## Known mecab-format dictionary projects + +| Dictionary | Language | Archive to download | Encoding of the files | +|---|---|---|---| +| IPADIC 2.7.0 | Japanese | `mecab-ipadic-2.7.0-20070801.tar.gz` from the MeCab project's SourceForge file area (`sourceforge.net/projects/mecab/files/mecab-ipadic/2.7.0-20070801/`) | EUC-JP | +| mecab-ko-dic 2.1.1 | Korean | `mecab-ko-dic-2.1.1-20180720.tar.gz` from the project's Bitbucket downloads page (`bitbucket.org/eunjeon/mecab-ko-dic/downloads/`) | UTF-8 | + +Both archives are gzip-compressed tars, the format `MecabDictionaryInstaller` reads. The encoding column matters: `MecabDictionary.load(Path)` assumes UTF-8, and a dictionary in any other encoding is loaded with the two-argument overload, for example `MecabDictionary.load(dir, Charset.forName("EUC-JP"))` for IPADIC. + +## Step 1: download the archive + +Either fetch the archive with any tool you like, or use the helper next to this file, which adds checksum verification: + +``` +./download-mecab-dictionary.sh ipadic.tar.gz [expected-sha256] +``` + +On the first run without a checksum the script prints the SHA-256 it computed; record it and pass it on later runs so a changed or corrupted download fails loudly instead of being installed. + +## Step 2: unpack with the installer + +The installer extracts only the dictionary payload: the `*.csv` and `*.def` files a `MecabDictionary` reads, plus the `dicrc` configuration file the distributions ship alongside them. It flattens the entries into the target directory, and by the same flattening makes it impossible for an archive path to escape that directory: + +```java +import java.nio.file.Path; +import opennlp.tools.tokenize.lattice.MecabDictionaryInstaller; + +int files = MecabDictionaryInstaller.install( + Path.of("ipadic.tar.gz").toUri(), Path.of("ipadic")); +``` + +The returned count is the number of dictionary files extracted. A remote URI works in the same call if you prefer to skip step 1 entirely and let the installer stream the download. + +## Step 3: load and tokenize + +```java +import java.nio.charset.Charset; +import java.nio.file.Path; +import opennlp.tools.tokenize.lattice.LatticeTokenizer; +import opennlp.tools.tokenize.lattice.MecabDictionary; + +MecabDictionary dictionary = + MecabDictionary.load(Path.of("ipadic"), Charset.forName("EUC-JP")); +LatticeTokenizer tokenizer = new LatticeTokenizer(dictionary); +// "Tokyo-to ni iku" (go to the Tokyo metropolis), escaped to keep this file ASCII +String[] tokens = tokenizer.tokenize("\u6771\u4EAC\u90FD\u306B\u884C\u304F"); +``` + +For a UTF-8 dictionary such as mecab-ko-dic, `MecabDictionary.load(Path.of("ko-dic"))` is enough. Loaded dictionaries and tokenizers are immutable and safe to share between threads, so load once and reuse. + +## Chinese: the unigram segmenter needs only a frequency lexicon + +`opennlp.tools.tokenize.lattice.UnigramSegmenter` does not use mecab dictionaries. It loads a plain text lexicon, one entry per line: the word, its count, and optionally a tag, separated by whitespace. Any word-frequency list you have the rights to use works: + +```java +import java.nio.file.Path; +import opennlp.tools.tokenize.lattice.UnigramSegmenter; + +UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt")); +// "wo laidao Beijing Tian'anmen" (I arrive at Beijing Tiananmen), escaped as above +String[] tokens = segmenter.tokenize("\u6211\u6765\u5230\u5317\u4EAC\u5929\u5B89\u95E8"); +``` + +As with the dictionaries, the lexicon carries its own license; nothing is bundled. diff --git a/dev/download-mecab-dictionary.sh b/dev/download-mecab-dictionary.sh new file mode 100755 index 0000000000..3b86ed09d6 --- /dev/null +++ b/dev/download-mecab-dictionary.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Downloads a mecab-format dictionary archive for the lattice tokenizer. +# +# This script only fetches the archive and, when a checksum is given, verifies it. +# It does not unpack anything: unpacking is the job of +# opennlp.tools.tokenize.lattice.MecabDictionaryInstaller, which extracts exactly the +# files a MecabDictionary reads and nothing else. See README-mecab-dictionaries.md in +# this directory for the known dictionary projects, their encodings, and the Java +# steps that follow the download. +# +# The dictionary you download carries its own license. Nothing is bundled with +# Apache OpenNLP and no dictionary location is built in. + +set -euo pipefail + +usage() { + echo "usage: $0 [expected-sha256]" >&2 + echo "" >&2 + echo " archive-url where to fetch the gzip-compressed tar from" >&2 + echo " target-file where to store the downloaded archive" >&2 + echo " expected-sha256 optional checksum; when given, the download is verified" >&2 + echo " and a mismatch deletes the file and fails the script" >&2 + exit 2 +} + +[ $# -lt 2 ] && usage +url="$1" +target="$2" +expected="${3:-}" + +# Download to a temporary name first so an interrupted transfer never leaves a +# half-written file at the target path. +tmp="${target}.download" +echo "downloading ${url}" +curl --fail --location --retry 3 --output "${tmp}" "${url}" + +actual="$(sha256sum "${tmp}" | cut -d' ' -f1)" +if [ -n "${expected}" ]; then + if [ "${actual}" != "${expected}" ]; then + rm -f "${tmp}" + echo "checksum mismatch: expected ${expected}" >&2 + echo " but got ${actual}" >&2 + exit 1 + fi + echo "checksum verified: ${actual}" +else + echo "sha256 of the download: ${actual}" + echo "record this value and pass it as the third argument next time to verify" +fi + +mv "${tmp}" "${target}" +echo "stored ${target}" +echo "" +echo "next: unpack and load it from Java; see README-mecab-dictionaries.md" diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java new file mode 100644 index 0000000000..649be0b80d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.util.ArrayList; +import java.util.List; + +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.tokenize.lattice.MecabDictionary.Category; +import opennlp.tools.tokenize.lattice.MecabDictionary.WordEntry; +import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; + +/** + * Dictionary-driven segmentation for languages written without spaces: a Viterbi + * search over the word lattice of a {@link MecabDictionary}, minimizing the sum of + * word costs and connection costs. This is the segmentation approach behind Japanese + * and Korean morphological analysis; the same decoder serves both, since the language + * lives entirely in the user-supplied dictionary. + * + *

Unknown text is handled through the dictionary's character categories: where the + * lexicon has no entry, or a category always invokes them, unknown-word candidates are + * generated per category template, grouping runs of same-category characters when the + * category says so. Whitespace never joins a morpheme and is never reported as one. + * Every reported span is in original text coordinates.

+ * + *

{@link #analyze(String)} returns full morphemes with their dictionary features; + * the {@link Tokenizer} view reports just the surfaces and spans.

+ * + *

The tokenizer reads only immutable dictionary state and is safe to share between + * threads.

+ * + * @since 3.0.0 + */ +public class LatticeTokenizer implements Tokenizer { + + /** The context id of the beginning and end of text. */ + private static final int BOUNDARY_CONTEXT = 0; + + private final MecabDictionary dictionary; + + /** + * Initializes the tokenizer. + * + * @param dictionary The dictionary to segment with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code dictionary} is {@code null}. + */ + public LatticeTokenizer(MecabDictionary dictionary) { + if (dictionary == null) { + throw new IllegalArgumentException("dictionary must not be null"); + } + this.dictionary = dictionary; + } + + /** + * One lattice node: a candidate morpheme with its best path cost so far. Nodes + * ending at one position chain through {@link #nextEndingHere}. + */ + private static final class Node { + private final int start; + private final int end; + private final WordEntry entry; + private final boolean unknown; + private long pathCost = Long.MAX_VALUE; + private Node previous; + private Node nextEndingHere; + + private Node(int start, int end, WordEntry entry, boolean unknown) { + this.start = start; + this.end = end; + this.entry = entry; + this.unknown = unknown; + } + } + + /** + * Segments a text into morphemes with their dictionary features. + * + * @param text The text to segment. Must not be {@code null}. + * @return The morphemes in text order, spans in original coordinates, whitespace + * omitted. Never {@code null}; empty for empty or all-whitespace input. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position, which a {@code unk.def} without a {@code DEFAULT} template does. + */ + public List analyze(String text) { + if (text == null) { + throw new IllegalArgumentException("text must not be null"); + } + final List morphemes = new ArrayList<>(); + int start = 0; + while (start < text.length()) { + if (StringUtil.isWhitespace(text.charAt(start))) { + start++; + continue; + } + int end = start; + while (end < text.length() && !StringUtil.isWhitespace(text.charAt(end))) { + end++; + } + decode(text, start, end, morphemes); + start = end; + } + return morphemes; + } + + /** + * {@inheritDoc} + * + *

Reports the segmented surfaces, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position; see {@link #analyze(String)}. + */ + @Override + public String[] tokenize(String text) { + final List morphemes = analyze(text); + final String[] tokens = new String[morphemes.size()]; + for (int i = 0; i < tokens.length; i++) { + tokens[i] = morphemes.get(i).surface(); + } + return tokens; + } + + /** + * {@inheritDoc} + * + *

Reports the segmented spans in original text coordinates, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position; see {@link #analyze(String)}. + */ + @Override + public Span[] tokenizePos(String text) { + final List morphemes = analyze(text); + final Span[] spans = new Span[morphemes.size()]; + for (int i = 0; i < spans.length; i++) { + spans[i] = morphemes.get(i).span(); + } + return spans; + } + + /** + * Runs the Viterbi search over one whitespace-free stretch of text. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param morphemes Receives the cheapest path's morphemes, in text order. + * @throws IllegalStateException Thrown if no path reaches the end of the stretch. + */ + private void decode(String text, int from, int to, List morphemes) { + final int length = to - from; + // Each element heads the chain of nodes ending at that position. + final Node[] endingAt = new Node[length + 1]; + + final Category[] categoryAt = new Category[length]; + final int[] runEndAt = new int[length]; + computeCategoryRuns(text, from, to, categoryAt, runEndAt); + + final List candidates = new ArrayList<>(); + for (int i = 0; i < length; i++) { + if (i > 0 && endingAt[i] == null) { + continue; + } + candidates.clear(); + candidates(text, from, to, i, categoryAt[i], runEndAt[i], candidates); + for (final Node candidate : candidates) { + relax(candidate, i == 0 ? null : endingAt[i]); + if (candidate.pathCost < Long.MAX_VALUE) { + final int end = candidate.end - from; + candidate.nextEndingHere = endingAt[end]; + endingAt[end] = candidate; + } + } + } + + Node best = null; + long bestTotal = Long.MAX_VALUE; + for (Node node = endingAt[length]; node != null; node = node.nextEndingHere) { + final long total = node.pathCost + + dictionary.connectionCost(node.entry.rightId(), BOUNDARY_CONTEXT); + if (best == null || total < bestTotal) { + best = node; + bestTotal = total; + } + } + if (best == null) { + throw new IllegalStateException( + "no segmentation path for \"" + text.subSequence(from, to) + "\""); + } + + final List reversed = new ArrayList<>(); + for (Node node = best; node != null; node = node.previous) { + reversed.add(new Morpheme(new Span(node.start, node.end), + text.substring(node.start, node.end), node.entry.features(), node.unknown)); + } + for (int i = reversed.size() - 1; i >= 0; i--) { + morphemes.add(reversed.get(i)); + } + } + + /** + * Connects a candidate to the cheapest predecessor ending where it starts. + * + * @param candidate The node to give a path cost and a predecessor. + * @param predecessors The head of the chain of nodes ending where the candidate + * starts, or {@code null} when it starts at the stretch start. + */ + private void relax(Node candidate, Node predecessors) { + if (predecessors == null) { + candidate.pathCost = candidate.entry.cost() + + dictionary.connectionCost(BOUNDARY_CONTEXT, candidate.entry.leftId()); + return; + } + for (Node predecessor = predecessors; predecessor != null; + predecessor = predecessor.nextEndingHere) { + final long total = predecessor.pathCost + + dictionary.connectionCost(predecessor.entry.rightId(), candidate.entry.leftId()) + + candidate.entry.cost(); + if (total < candidate.pathCost) { + candidate.pathCost = total; + candidate.previous = predecessor; + } + } + } + + /** + * Fills the per-position category and same-category run end for one stretch, in one + * right-to-left pass over its code points. Positions inside a surrogate pair keep a + * {@code null} category; no candidate ever starts there. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param categoryAt Receives each position's category, indexed by {@code + * position - from}. + * @param runEndAt Receives each position's exclusive same-category run end, indexed + * the same way. + */ + private void computeCategoryRuns(String text, int from, int to, + Category[] categoryAt, int[] runEndAt) { + int next = -1; + for (int position = to; position > from; ) { + final int codePoint = text.codePointBefore(position); + position -= Character.charCount(codePoint); + final int index = position - from; + categoryAt[index] = dictionary.categoryOf(codePoint); + if (next >= 0 && categoryAt[next] == categoryAt[index]) { + runEndAt[index] = runEndAt[next]; + } else { + runEndAt[index] = next >= 0 ? next + from : to; + } + next = index; + } + } + + /** + * Gathers lexicon matches and unknown-word candidates starting at one position. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end, which no candidate may reach past. + * @param offset The candidate start, relative to {@code from}. + * @param positionCategory The category of that position, or {@code null} for a + * position inside a surrogate pair. + * @param positionRunEnd The exclusive end of the same-category run starting there, + * meaningful only when {@code positionCategory} is not + * {@code null}. + * @param candidates Receives the candidates. Must be empty on entry. + * @throws IllegalStateException Thrown if neither the lexicon, the position's + * category, nor the {@code DEFAULT} template offers a candidate. + */ + private void candidates(String text, int from, int to, int offset, + Category positionCategory, int positionRunEnd, List candidates) { + final int position = from + offset; + dictionary.prefixMatches(text, position, to, (length, entries) -> { + for (final WordEntry entry : entries) { + candidates.add(new Node(position, position + length, entry, false)); + } + }); + final boolean lexiconMatch = !candidates.isEmpty(); + + final int codePoint = text.codePointAt(position); + final Category category; + final int runEnd; + if (positionCategory == null) { + // Only a lexicon surface ending inside a surrogate pair can make such a + // position reachable; classify the stray code unit on the spot so the lattice + // stays connected. + category = dictionary.categoryOf(codePoint); + runEnd = position + Character.charCount(codePoint); + } else { + category = positionCategory; + runEnd = positionRunEnd; + } + if (!lexiconMatch || category.invoke()) { + final List templates = dictionary.unknownEntries(category.name()); + if (templates != null) { + addUnknown(candidates, text, position, runEnd, category, templates); + } + } + if (candidates.isEmpty()) { + // Neither the lexicon nor the character's category produced a candidate here, so a + // single-character entry from the DEFAULT template keeps the lattice connected. + final List fallback = + dictionary.unknownEntries(MecabDictionary.DEFAULT_CATEGORY); + if (fallback != null) { + for (final WordEntry entry : fallback) { + candidates.add( + new Node(position, position + Character.charCount(codePoint), entry, true)); + } + } + } + if (candidates.isEmpty()) { + throw new IllegalStateException("dictionary provides no candidate at position " + + position + "; unk.def lacks a DEFAULT template"); + } + } + + /** + * Emits unknown-word candidates per the category's grouping and length settings. + * + *

Every candidate stays inside the same-category run, so an unknown word never + * glues characters of different categories together, and every length counts whole + * characters rather than code units.

+ * + * @param candidates Receives the candidates. + * @param text The text being segmented. + * @param position The position the candidates start at. + * @param runEnd The exclusive end of the same-category run starting at + * {@code position}. + * @param category The category of that run. + * @param templates The category's unknown-word templates. + */ + private void addUnknown(List candidates, String text, int position, + int runEnd, Category category, List templates) { + if (category.group()) { + for (final WordEntry entry : templates) { + candidates.add(new Node(position, runEnd, entry, true)); + } + } + final int lengths = category.length(); + int end = position; + for (int length = 1; length <= lengths && end < runEnd; length++) { + end += Character.charCount(text.codePointAt(end)); + if (category.group() && end == runEnd) { + // This length coincides with the grouped run emitted above; skip the duplicate. + continue; + } + for (final WordEntry entry : templates) { + candidates.add(new Node(position, end, entry, true)); + } + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java new file mode 100644 index 0000000000..59d9ab3388 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -0,0 +1,907 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.util.StringUtil; + +/** + * An immutable, in-memory dictionary in the mecab directory format: lexicon entries + * from the {@code *.csv} files, connection costs from {@code matrix.def}, character + * categories from {@code char.def}, and unknown-word templates from {@code unk.def}, + * loaded from a user-supplied dictionary directory. No dictionary data is bundled or + * downloaded by this class. + * + *

The same format serves multiple languages: the Japanese IPADIC and UniDic + * distributions and the Korean mecab-ko-dic all load through this one reader, with the + * feature columns passed through untouched because their schemas differ.

+ * + *

Instances are immutable and safe to share between threads.

+ * + * @see LatticeTokenizer + * @since 3.0.0 + */ +public final class MecabDictionary { + + /** + * The category name every {@code char.def} must define; unmapped code points and + * unknown-word handling fall back to it. + */ + static final String DEFAULT_CATEGORY = "DEFAULT"; + + private static final String MATRIX_DEF = "matrix.def"; + private static final String CHAR_DEF = "char.def"; + private static final String UNK_DEF = "unk.def"; + + /** The prefix a {@code char.def} code point field carries, in either letter case. */ + private static final String HEX_PREFIX = "0x"; + + /** The separator between the two ends of a {@code char.def} code point range. */ + private static final String RANGE_SEPARATOR = ".."; + + /** The {@code char.def} field value that turns a category flag on. */ + private static final String FLAG_ON = "1"; + + /** + * One lexicon or unknown-word entry. + * + * @param leftId The left context id, an index into the connection matrix. + * @param rightId The right context id, an index into the connection matrix. + * @param cost The entry's own cost. + * @param features The entry's feature columns, in file order. + */ + record WordEntry(int leftId, int rightId, int cost, List features) { + } + + /** + * One character category's unknown-word behavior from {@code char.def}. + * + * @param name The category name. + * @param invoke Whether unknown-word candidates are generated even where the lexicon + * matched. + * @param group Whether a whole run of same-category characters is offered as one + * candidate. + * @param length How many leading characters of the run are offered as candidates. + */ + record Category(String name, boolean invoke, boolean group, int length) { + } + + /** + * The lexicon as a double-array trie: one transition is one array read and one + * comparison. Characters are recoded into dense labels ordered by descending + * frequency before the array is built, which keeps the array compact; a character + * the lexicon never uses misses in the recode table before the array is consulted. + * + *

The layout is the classic base/check pair: from state {@code s}, label + * {@code c} leads to {@code t = base[s] + c} exactly when {@code check[t] == s}. + * Label {@code 0} terminates a surface and leads to a state whose negative base + * encodes the index of the surface's entry list.

+ */ + private static final class DoubleArrayLexicon { + + private final int[] base; + private final int[] check; + private final int[] codeOf; + private final List[] values; + + private DoubleArrayLexicon(int[] base, int[] check, int[] codeOf, + List[] values) { + this.base = base; + this.check = check; + this.codeOf = codeOf; + this.values = values; + } + + /** + * Builds the trie from the surface-keyed lexicon. + * + * @param lexicon The entries keyed by surface form. + * @return The built trie. Never {@code null}. + */ + @SuppressWarnings("unchecked") + private static DoubleArrayLexicon build(Map> lexicon) { + final String[] surfaces = lexicon.keySet().toArray(new String[0]); + Arrays.sort(surfaces); + final List[] values = new List[surfaces.length]; + for (int i = 0; i < surfaces.length; i++) { + values[i] = List.copyOf(lexicon.get(surfaces[i])); + } + + // Dense recode: labels ordered by descending frequency get the small codes, so + // busy transitions cluster at the low end of the array. + final int[] frequency = new int[Character.MAX_VALUE + 1]; + for (final String surface : surfaces) { + for (int i = 0; i < surface.length(); i++) { + frequency[surface.charAt(i)]++; + } + } + final Integer[] chars = new Integer[Character.MAX_VALUE + 1]; + int distinct = 0; + for (int c = 0; c <= Character.MAX_VALUE; c++) { + if (frequency[c] > 0) { + chars[distinct++] = c; + } + } + final Integer[] ordered = Arrays.copyOf(chars, distinct); + Arrays.sort(ordered, (a, b) -> frequency[b] - frequency[a]); + final int[] codeOf = new int[Character.MAX_VALUE + 1]; + Arrays.fill(codeOf, -1); + for (int rank = 0; rank < ordered.length; rank++) { + codeOf[ordered[rank]] = rank + 1; + } + + final Builder builder = new Builder(surfaces, codeOf); + builder.insert(0, surfaces.length, 0, Builder.ROOT); + return new DoubleArrayLexicon(Arrays.copyOf(builder.base, builder.high + 1), + Arrays.copyOf(builder.check, builder.high + 1), codeOf, values); + } + + /** + * Reports every surface starting at a text position, walking the array once. + * + * @param text The text being segmented. + * @param from The position surfaces must start at. + * @param to The exclusive end of the searchable stretch. + * @param consumer Receives each match length with its entries. + */ + private void prefixMatches(String text, int from, int to, + PrefixMatchConsumer consumer) { + int state = Builder.ROOT; + for (int i = from; i < to; i++) { + final char c = text.charAt(i); + final int code = codeOf[c]; + if (code < 0) { + return; + } + final int next = base[state] + code; + if (next >= check.length || check[next] != state) { + return; + } + state = next; + final int terminal = base[state]; + if (terminal < check.length && check[terminal] == state && base[terminal] < 0) { + consumer.accept(i - from + 1, values[-base[terminal] - 1]); + } + } + } + + /** + * The recursive sorted-range builder: each call places one node's children by + * finding a base at which every child label lands on a free slot, then recurses + * per child range. A moving watermark keeps the free-slot search near-linear over + * real lexicons. + */ + private static final class Builder { + + private static final int ROOT = 1; + private static final int EMPTY = -1; + + private final String[] surfaces; + private final int[] codeOf; + private int[] base; + private int[] check; + private int high = ROOT; + private int watermark = ROOT + 1; + private int valueIndex; + + private Builder(String[] surfaces, int[] codeOf) { + this.surfaces = surfaces; + this.codeOf = codeOf; + base = new int[1 << 16]; + check = new int[1 << 16]; + Arrays.fill(check, EMPTY); + } + + /** + * Places the children of one trie node. + * + * @param left The first surface of the node's range. + * @param right The exclusive last surface of the node's range. + * @param depth The character depth of the node. + * @param state The node's own slot. + */ + private void insert(int left, int right, int depth, int state) { + // gather the distinct child labels of this range, terminator first + final int[] labels = new int[right - left]; + int labelCount = 0; + int previous = -2; + for (int k = left; k < right; k++) { + final int label = surfaces[k].length() == depth + ? 0 : codeOf[surfaces[k].charAt(depth)]; + if (label != previous) { + labels[labelCount++] = label; + previous = label; + } + } + final int found = findBase(labels, labelCount); + base[state] = found; + for (int k = 0; k < labelCount; k++) { + final int child = found + labels[k]; + check[child] = state; + if (child > high) { + high = child; + } + } + // recurse over each child's sub-range + int start = left; + for (int k = 0; k < labelCount; k++) { + final int label = labels[k]; + int end = start; + while (end < right && (surfaces[end].length() == depth + ? 0 : codeOf[surfaces[end].charAt(depth)]) == label) { + end++; + } + final int child = found + label; + if (label == 0) { + base[child] = -(++valueIndex); + } else { + insert(start, end, depth + 1, child); + } + start = end; + } + } + + /** + * Finds the lowest base at which every label lands on a free slot. Labels + * arrive in surface-character order, not numeric order, so the smallest and + * largest label are computed rather than assumed positional. + * + * @param labels The child labels to place. + * @param labelCount How many leading elements of {@code labels} are in use. + * @return The base offset every label fits at. + */ + private int findBase(int[] labels, int labelCount) { + int smallest = labels[0]; + int largest = labels[0]; + for (int k = 1; k < labelCount; k++) { + smallest = Math.min(smallest, labels[k]); + largest = Math.max(largest, labels[k]); + } + int candidate = Math.max(1, watermark - smallest); + while (true) { + ensureCapacity(candidate + largest); + boolean fits = true; + for (int k = 0; fits && k < labelCount; k++) { + fits = check[candidate + labels[k]] == EMPTY; + } + if (fits) { + while (watermark < check.length && check[watermark] != EMPTY) { + watermark++; + } + return candidate; + } + candidate++; + } + } + + /** + * Grows the base and check arrays until a slot is addressable. + * + * @param slot The highest slot index that has to be writable. + */ + private void ensureCapacity(int slot) { + if (slot >= check.length) { + int capacity = check.length; + while (capacity <= slot) { + capacity += capacity >> 1; + } + base = Arrays.copyOf(base, capacity); + final int old = check.length; + check = Arrays.copyOf(check, capacity); + Arrays.fill(check, old, capacity, EMPTY); + } + } + } + } + + /** + * The {@code char.def} code point to category name mapping, over the whole Unicode + * code point range. + * + *

The Basic Multilingual Plane is held in a directly indexed array. The + * supplementary planes are held as a sorted, non-overlapping range table searched by + * binary search, because dictionaries map them in a handful of large blocks.

+ */ + private static final class CategoryTable { + + private final Category[] bmp; + private final int[] rangeStart; + private final int[] rangeEnd; + private final Category[] rangeCategory; + + private CategoryTable(Category[] bmp, int[] rangeStart, int[] rangeEnd, + Category[] rangeCategory) { + this.bmp = bmp; + this.rangeStart = rangeStart; + this.rangeEnd = rangeEnd; + this.rangeCategory = rangeCategory; + } + + /** + * Looks up the category a {@code char.def} mapping gives a code point. The table + * holds the {@link Category} instances themselves, and two code points of one + * category share one instance, so categories may be compared by identity. + * + * @param codePoint The code point to classify. + * @return The category, or {@code null} when no mapping covers the code point. + */ + private Category categoryOf(int codePoint) { + if (codePoint <= Character.MAX_VALUE) { + return bmp[codePoint]; + } + int low = 0; + int high = rangeStart.length - 1; + while (low <= high) { + final int middle = (low + high) >>> 1; + if (codePoint < rangeStart[middle]) { + high = middle - 1; + } else if (codePoint > rangeEnd[middle]) { + low = middle + 1; + } else { + return rangeCategory[middle]; + } + } + return null; + } + } + + /** + * Collects {@code char.def} mappings in file order and folds them into a + * {@link CategoryTable}, giving a later mapping precedence over an earlier one that + * covers the same code point, which is what direct indexing does for the BMP. + */ + private static final class CategoryTableBuilder { + + private final String[] bmp = new String[Character.MAX_VALUE + 1]; + private final List bounds = new ArrayList<>(); + private final List names = new ArrayList<>(); + + /** + * Records one inclusive code point range's category. + * + * @param from The first code point of the range. + * @param to The last code point of the range, inclusive. + * @param category The category name to give the range. Must not be {@code null}. + */ + private void map(int from, int to, String category) { + for (int c = from; c <= Math.min(to, Character.MAX_VALUE); c++) { + bmp[c] = category; + } + if (to > Character.MAX_VALUE) { + bounds.add(new int[] {Math.max(from, Character.MAX_VALUE + 1), to}); + names.add(category); + } + } + + /** + * Folds the recorded mappings into their lookup table. + * + * @param categories The categories the {@code char.def} category section defined, + * keyed by name. + * @return The table. Never {@code null}. + * @throws IOException Thrown if a mapping names a category that was never defined. + */ + private CategoryTable build(Map categories) throws IOException { + // Cut the supplementary ranges at every boundary they introduce, so that each + // resulting elementary interval is covered by a single winning range and the + // table stays sorted and non-overlapping for binary search. + final int[] edges = new int[bounds.size() * 2]; + for (int i = 0; i < bounds.size(); i++) { + edges[i * 2] = bounds.get(i)[0]; + edges[i * 2 + 1] = bounds.get(i)[1] + 1; + } + Arrays.sort(edges); + final List intervals = new ArrayList<>(); + final List winners = new ArrayList<>(); + for (int i = 0; i < edges.length - 1; i++) { + if (edges[i] == edges[i + 1]) { + continue; + } + final String winner = lastCovering(edges[i]); + if (winner == null) { + continue; + } + final int previous = intervals.size() - 1; + if (previous >= 0 && intervals.get(previous)[1] == edges[i] - 1 + && winners.get(previous).equals(winner)) { + intervals.get(previous)[1] = edges[i + 1] - 1; + } else { + intervals.add(new int[] {edges[i], edges[i + 1] - 1}); + winners.add(winner); + } + } + final int[] starts = new int[intervals.size()]; + final int[] ends = new int[intervals.size()]; + for (int i = 0; i < intervals.size(); i++) { + starts[i] = intervals.get(i)[0]; + ends[i] = intervals.get(i)[1]; + } + final Category[] resolvedBmp = new Category[bmp.length]; + for (int c = 0; c < bmp.length; c++) { + if (bmp[c] != null) { + resolvedBmp[c] = resolve(bmp[c], categories, c); + } + } + final Category[] resolvedRanges = new Category[winners.size()]; + for (int i = 0; i < winners.size(); i++) { + resolvedRanges[i] = resolve(winners.get(i), categories, starts[i]); + } + return new CategoryTable(resolvedBmp, starts, ends, resolvedRanges); + } + + /** + * Resolves a mapped category name against the defined categories, so a mapping to + * a name the {@code char.def} category section never defined fails at load with + * the offending code point instead of falling back silently at lookup time. + * + * @param name The category name a mapping line gave. + * @param categories The defined categories, keyed by name. + * @param codePoint A code point the mapping covers, for the error message. + * @return The resolved category. Never {@code null}. + * @throws IOException Thrown if no category of that name was defined. + */ + private Category resolve(String name, Map categories, + int codePoint) throws IOException { + final Category category = categories.get(name); + if (category == null) { + throw new IOException(String.format( + CHAR_DEF + " maps U+%04X to the undefined category %s", codePoint, name)); + } + return category; + } + + /** + * Finds the category of the last recorded range covering a code point. + * + * @param codePoint The code point to look up. + * @return The category name, or {@code null} when no recorded range covers it. + */ + private String lastCovering(int codePoint) { + for (int i = bounds.size() - 1; i >= 0; i--) { + final int[] range = bounds.get(i); + if (codePoint >= range[0] && codePoint <= range[1]) { + return names.get(i); + } + } + return null; + } + } + + /** Receives one common-prefix match during {@link #prefixMatches}. */ + interface PrefixMatchConsumer { + + /** + * Accepts one match. + * + * @param length The matched surface length in characters. + * @param entries The lexicon entries for that surface. + */ + void accept(int length, List entries); + } + + private final DoubleArrayLexicon lexicon; + private final short[] connectionCosts; + private final int rightSize; + private final CategoryTable categoryTable; + private final Category defaultCategory; + private final Map> unknownEntries; + + private MecabDictionary(DoubleArrayLexicon lexicon, + short[] connectionCosts, int rightSize, Map categories, + CategoryTable categoryTable, Map> unknownEntries) { + this.lexicon = lexicon; + this.connectionCosts = connectionCosts; + this.rightSize = rightSize; + this.categoryTable = categoryTable; + this.defaultCategory = categories.get(DEFAULT_CATEGORY); + this.unknownEntries = unknownEntries; + } + + /** + * Loads a dictionary directory encoded in UTF-8. + * + * @param directory The unpacked dictionary directory. Must not be {@code null}. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if reading fails or a file is malformed. + * @throws IllegalArgumentException Thrown if {@code directory} is {@code null}. + */ + public static MecabDictionary load(Path directory) throws IOException { + return load(directory, StandardCharsets.UTF_8); + } + + /** + * Loads a dictionary directory. + * + * @param directory The unpacked dictionary directory holding the {@code *.csv} + * lexicon files, {@code matrix.def}, {@code char.def}, and + * {@code unk.def}. Must not be {@code null}. + * @param charset The encoding the distribution uses, for example UTF-8 or EUC-JP. + * Must not be {@code null}. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if reading fails, a required file is missing, a file is + * malformed, or a lexicon entry's context ids are outside the + * {@code matrix.def} dimensions. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static MecabDictionary load(Path directory, Charset charset) throws IOException { + if (directory == null) { + throw new IllegalArgumentException("directory must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); + } + // The connection matrix is read first because its dimensions are what every + // lexicon entry's context ids have to be inside of. + final List matrixLines = readLines(directory.resolve(MATRIX_DEF), charset); + final String headerLine = StringUtil.trimUnicodeWhitespace(matrixLines.get(0)); + if (headerLine.isEmpty()) { + throw new IOException("empty " + MATRIX_DEF + " under " + directory); + } + final String[] header = splitWhitespace(headerLine); + if (header.length != 2) { + throw new IOException("malformed " + MATRIX_DEF + " header: " + headerLine); + } + final int leftSize = parseInt(header[0], MATRIX_DEF, 1); + final int rightSize = parseInt(header[1], MATRIX_DEF, 1); + if (leftSize < 1 || rightSize < 1) { + throw new IOException(MATRIX_DEF + " dimensions must be positive, got " + + leftSize + " " + rightSize); + } + final long cells = (long) leftSize * rightSize; + if (cells > Integer.MAX_VALUE) { + throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize + + " overflow the addressable connection matrix"); + } + final short[] costs = new short[(int) cells]; + for (int i = 1; i < matrixLines.size(); i++) { + final String line = StringUtil.trimUnicodeWhitespace(matrixLines.get(i)); + if (line.isEmpty()) { + continue; + } + final String[] fields = splitWhitespace(line); + if (fields.length != 3) { + throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1)); + } + final int right = parseInt(fields[0], MATRIX_DEF, i + 1); + final int left = parseInt(fields[1], MATRIX_DEF, i + 1); + if (right < 0 || right >= leftSize || left < 0 || left >= rightSize) { + throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1) + + ": context ids " + right + " " + left + + " are outside the declared dimensions " + leftSize + " " + rightSize); + } + final int cost = parseInt(fields[2], MATRIX_DEF, i + 1); + if (cost < Short.MIN_VALUE || cost > Short.MAX_VALUE) { + throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1) + + ": connection cost " + cost + " is outside the 16-bit range the" + + " format defines"); + } + costs[right * rightSize + left] = (short) cost; + } + + final Map> lexicon = new HashMap<>(); + try (DirectoryStream csvFiles = Files.newDirectoryStream(directory, "*.csv")) { + for (final Path csv : csvFiles) { + readLexicon(csv, charset, lexicon, leftSize, rightSize); + } + } + if (lexicon.isEmpty()) { + throw new IOException("no lexicon entries found under " + directory); + } + + final Map categories = new HashMap<>(); + final CategoryTableBuilder categoryTable = new CategoryTableBuilder(); + readCharacterDefinition(directory.resolve(CHAR_DEF), charset, categories, + categoryTable); + final Map> unknown = new HashMap<>(); + readLexicon(directory.resolve(UNK_DEF), charset, unknown, leftSize, rightSize); + + return new MecabDictionary(DoubleArrayLexicon.build(lexicon), costs, + rightSize, categories, categoryTable.build(categories), unknown); + } + + /** + * Reads one lexicon-format CSV file, rejecting any entry whose context ids the + * connection matrix cannot be indexed with. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @param target Receives the entries, keyed by surface form. + * @param leftSize The first {@code matrix.def} dimension, which bounds right context + * ids. + * @param rightSize The second {@code matrix.def} dimension, which bounds left context + * ids. + * @throws IOException Thrown if the file is missing, an entry is malformed, or an + * entry's context id is outside the matrix dimensions. + */ + private static void readLexicon(Path file, Charset charset, + Map> target, int leftSize, int rightSize) + throws IOException { + int lineNumber = 0; + for (final String line : readLines(file, charset)) { + lineNumber++; + if (line.isEmpty()) { + continue; + } + final List fields = splitCsv(line); + if (fields.size() < 4) { + throw new IOException("malformed entry at " + file + " line " + lineNumber); + } + final String surface = fields.get(0); + if (surface.isEmpty()) { + continue; + } + final int leftId = parseInt(fields.get(1), file.toString(), lineNumber); + final int rightId = parseInt(fields.get(2), file.toString(), lineNumber); + if (leftId < 0 || leftId >= rightSize) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": left context id " + leftId + " is outside the " + MATRIX_DEF + " dimensions " + + leftSize + " " + rightSize); + } + if (rightId < 0 || rightId >= leftSize) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": right context id " + rightId + " is outside the " + MATRIX_DEF + " dimensions " + + leftSize + " " + rightSize); + } + final WordEntry entry = new WordEntry(leftId, rightId, + parseInt(fields.get(3), file.toString(), lineNumber), + List.copyOf(fields.subList(4, fields.size()))); + target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); + } + } + + /** + * Reads {@code char.def}: the category behavior lines and the code point mapping + * lines, in file order, so that a later mapping wins over an earlier one. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @param categories Receives the defined categories, keyed by name. + * @param categoryTable Receives the code point to category name mappings. + * @throws IOException Thrown if the file is missing, a line is malformed, a code + * point is outside the Unicode range, a range descends, or the file defines + * no {@code DEFAULT} category. + */ + private static void readCharacterDefinition(Path file, Charset charset, + Map categories, CategoryTableBuilder categoryTable) + throws IOException { + int lineNumber = 0; + for (final String raw : readLines(file, charset)) { + lineNumber++; + final String line = StringUtil.trimUnicodeWhitespace(stripComment(raw)); + if (line.isEmpty()) { + continue; + } + final String[] fields = splitWhitespace(line); + if (fields[0].regionMatches(true, 0, HEX_PREFIX, 0, HEX_PREFIX.length())) { + final int rangeSeparator = fields[0].indexOf(RANGE_SEPARATOR); + final int from; + final int to; + if (rangeSeparator >= 0) { + from = parseCodePoint(fields[0].substring(0, rangeSeparator), file, lineNumber); + to = parseCodePoint( + fields[0].substring(rangeSeparator + RANGE_SEPARATOR.length()), file, lineNumber); + } else { + from = parseCodePoint(fields[0], file, lineNumber); + to = from; + } + if (fields.length < 2) { + throw new IOException("mapping without category at " + file + " line " + lineNumber); + } + if (from > to) { + throw new IOException("code point range descends at " + file + " line " + + lineNumber); + } + categoryTable.map(from, to, fields[1]); + } else { + if (fields.length < 4) { + throw new IOException("malformed category at " + file + " line " + lineNumber); + } + categories.put(fields[0], new Category(fields[0], + FLAG_ON.equals(fields[1]), FLAG_ON.equals(fields[2]), + parseInt(fields[3], file.toString(), lineNumber))); + } + } + if (!categories.containsKey(DEFAULT_CATEGORY)) { + throw new IOException(CHAR_DEF + " defines no " + DEFAULT_CATEGORY + " category: " + file); + } + } + + /** + * Reports every lexicon surface starting at a text position, walking the trie once + * with no substring allocation. + * + * @param text The text being segmented. + * @param from The position surfaces must start at. + * @param to The exclusive end of the searchable stretch. + * @param consumer Receives each match. + */ + void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) { + lexicon.prefixMatches(text, from, to, consumer); + } + + /** + * Reads the connection cost between two adjacent nodes. + * + * @param rightId The right context id of the earlier node. + * @param leftId The left context id of the later node. + * @return The connection cost. + */ + int connectionCost(int rightId, int leftId) { + return connectionCosts[rightId * rightSize + leftId]; + } + + /** + * Classifies a character by code point, so that a character outside the Basic + * Multilingual Plane is classified as the one character it is rather than as its two + * surrogates. + * + * @param codePoint The code point to classify. + * @return Its category, falling back to {@code DEFAULT} when no {@code char.def} + * mapping covers the code point. Never {@code null}. + */ + Category categoryOf(int codePoint) { + final Category category = categoryTable.categoryOf(codePoint); + return category != null ? category : defaultCategory; + } + + /** + * Looks up the unknown-word templates of a category. + * + * @param category The category name. + * @return The templates, or {@code null} when the category has none. + */ + List unknownEntries(String category) { + return unknownEntries.get(category); + } + + /** + * Reads a required dictionary file into its lines, accepting LF and CRLF endings. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @return The lines without their line terminators. Never {@code null} and never + * empty: an empty file reads as one empty line. + * @throws IOException Thrown if the file is missing or reading fails. + */ + private static List readLines(Path file, Charset charset) throws IOException { + if (!Files.exists(file)) { + throw new IOException("required dictionary file is missing: " + file); + } + final String content = new String(Files.readAllBytes(file), charset); + final List lines = new ArrayList<>(); + int start = 0; + for (int i = 0; i <= content.length(); i++) { + if (i == content.length() || content.charAt(i) == '\n') { + int end = i; + if (end > start && content.charAt(end - 1) == '\r') { + end--; + } + lines.add(content.substring(start, end)); + start = i + 1; + } + } + return lines; + } + + /** + * Removes a trailing {@code #} comment from a {@code char.def} line. + * + * @param line The raw line. + * @return The line up to but excluding the first {@code #}, or the whole line when + * there is none. + */ + private static String stripComment(String line) { + final int hash = line.indexOf('#'); + return hash < 0 ? line : line.substring(0, hash); + } + + /** + * Splits a lexicon line at every comma. Surfaces containing commas are not + * representable, matching the plain-text lexicon format. + * + * @param line The line to split. + * @return The fields in order, empty fields included. Never {@code null}. + */ + private static List splitCsv(String line) { + final List fields = new ArrayList<>(); + int start = 0; + for (int i = 0; i <= line.length(); i++) { + if (i == line.length() || line.charAt(i) == ',') { + fields.add(line.substring(start, i)); + start = i + 1; + } + } + return fields; + } + + /** + * Splits a line into its whitespace-separated fields. + * + * @param line The line to split. + * @return The non-empty fields in order. Never {@code null}. + */ + private static String[] splitWhitespace(String line) { + final List parts = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= line.length(); i++) { + if (i == line.length() || StringUtil.isWhitespace(line.charAt(i))) { + if (start >= 0) { + parts.add(line.substring(start, i)); + start = -1; + } + } else if (start < 0) { + start = i; + } + } + return parts.toArray(new String[0]); + } + + /** + * Parses a decimal integer field, reporting the file and line on failure. + * + * @param text The field text. + * @param file The file being read, for the error message. + * @param lineNumber The line being read, for the error message. + * @return The parsed value. + * @throws IOException Thrown if the field is not a valid integer. + */ + private static int parseInt(String text, String file, int lineNumber) + throws IOException { + try { + return Integer.parseInt(StringUtil.trimUnicodeWhitespace(text)); + } catch (NumberFormatException e) { + throw new IOException("malformed number in " + file + " line " + lineNumber, e); + } + } + + /** + * Parses a {@code 0x}-prefixed hexadecimal code point from {@code char.def}. + * + * @param text The field text including the {@code 0x} prefix. + * @param file The file being read, for the error message. + * @param lineNumber The line being read, for the error message. + * @return The parsed code point, which may be in a supplementary plane. + * @throws IOException Thrown if the field is shorter than the prefix, is not a valid + * hexadecimal number, or names a value no Unicode code point has. + */ + private static int parseCodePoint(String text, Path file, int lineNumber) + throws IOException { + final int codePoint; + try { + codePoint = Integer.parseInt( + StringUtil.trimUnicodeWhitespace(text).substring(HEX_PREFIX.length()), 16); + } catch (RuntimeException e) { + throw new IOException("malformed code point in " + file + " line " + lineNumber, e); + } + if (!Character.isValidCodePoint(codePoint)) { + throw new IOException("code point out of range in " + file + " line " + lineNumber); + } + return codePoint; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java new file mode 100644 index 0000000000..76212b1b5d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.zip.GZIPInputStream; + +/** + * Fetches and unpacks a mecab-format dictionary archive into a local directory, so the + * dictionary is acquired by the user at install time and never ships with this library. + * The archive location is user-supplied; no dictionary data is bundled and no download + * location is built in. + * + *

The installer reads gzip-compressed tar archives, the format the common + * distributions use, and extracts only the dictionary payload: the {@code *.csv} + * lexicon files and {@code *.def} definition files that a {@link MecabDictionary} + * reads, plus the {@code dicrc} configuration file distributions ship alongside them. + * Entries are flattened to their base names, which also means no archive path can + * escape the target directory.

+ * + * @since 3.0.0 + */ +public final class MecabDictionaryInstaller { + + private static final int TAR_BLOCK = 512; + private static final int TAR_NAME_LENGTH = 100; + private static final int TAR_SIZE_OFFSET = 124; + private static final int TAR_SIZE_LENGTH = 12; + private static final int TAR_TYPE_OFFSET = 156; + + private MecabDictionaryInstaller() { + // This class exposes only static methods and is never instantiated. + } + + /** + * Downloads a dictionary archive and unpacks it. + * + * @param archive The archive location, a gzip-compressed tar. Must not be + * {@code null}. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if fetching, reading, or writing fails, or the archive + * contains no dictionary file. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or + * {@code archive} is not an absolute URI. + */ + public static int install(URI archive, Path targetDirectory) throws IOException { + if (archive == null) { + throw new IllegalArgumentException("archive must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); + } + try (InputStream in = archive.toURL().openStream()) { + return extract(in, targetDirectory); + } + } + + /** + * Unpacks a dictionary archive stream. + * + * @param archiveStream The gzip-compressed tar content. Must not be {@code null}. + * Not closed. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if reading or writing fails, or the archive contains no + * dictionary file. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static int extract(InputStream archiveStream, Path targetDirectory) + throws IOException { + if (archiveStream == null) { + throw new IllegalArgumentException("archiveStream must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); + } + Files.createDirectories(targetDirectory); + final InputStream tar = new GZIPInputStream(archiveStream); + final byte[] header = new byte[TAR_BLOCK]; + int extracted = 0; + while (readBlock(tar, header)) { + if (isEndBlock(header)) { + break; + } + final String name = headerName(header); + final long size = headerSize(header); + final char type = (char) header[TAR_TYPE_OFFSET]; + final String baseName = baseName(name); + final boolean wanted = (type == '0' || type == 0) + && (baseName.endsWith(".csv") || baseName.endsWith(".def") + || "dicrc".equals(baseName)); + if (wanted) { + final Path file = targetDirectory.resolve(baseName); + try (InputStream entry = boundedStream(tar, size)) { + Files.copy(entry, file, StandardCopyOption.REPLACE_EXISTING); + } + extracted++; + skip(tar, padding(size)); + } else { + skip(tar, size + padding(size)); + } + } + if (extracted == 0) { + throw new IOException("the archive contains no dictionary file"); + } + return extracted; + } + + /** + * Fills one tar block from the stream. + * + * @param in The tar stream. + * @param block The block buffer to fill completely. + * @return {@code true} when a full block was read, {@code false} at a clean end of + * stream before any byte of the block. + * @throws IOException Thrown if the stream ends inside the block or reading fails. + */ + private static boolean readBlock(InputStream in, byte[] block) throws IOException { + int filled = 0; + while (filled < block.length) { + final int read = in.read(block, filled, block.length - filled); + if (read < 0) { + if (filled == 0) { + return false; + } + throw new IOException("truncated tar header"); + } + filled += read; + } + return true; + } + + /** + * Recognizes the all-zero block that terminates a tar archive. + * + * @param block The block to inspect. + * @return {@code true} when every byte is zero. + */ + private static boolean isEndBlock(byte[] block) { + for (final byte b : block) { + if (b != 0) { + return false; + } + } + return true; + } + + /** + * Reads the NUL-terminated entry name from a tar header block. + * + * @param header The header block. + * @return The entry name. Never {@code null}. + */ + private static String headerName(byte[] header) { + int end = 0; + while (end < TAR_NAME_LENGTH && header[end] != 0) { + end++; + } + return new String(header, 0, end, StandardCharsets.UTF_8); + } + + /** + * Reads the octal entry size from a tar header block. + * + * @param header The header block. + * @return The entry size in bytes. + * @throws IOException Thrown if the size field holds a non-octal digit. + */ + private static long headerSize(byte[] header) throws IOException { + long size = 0; + for (int i = TAR_SIZE_OFFSET; i < TAR_SIZE_OFFSET + TAR_SIZE_LENGTH; i++) { + final byte b = header[i]; + if (b == 0 || b == ' ') { + continue; + } + if (b < '0' || b > '7') { + throw new IOException("malformed tar size field"); + } + size = size * 8 + (b - '0'); + } + return size; + } + + /** + * Strips any directory prefix from an archive entry name. + * + * @param name The entry name as stored in the archive. + * @return The part after the last {@code /}, or the whole name when there is none. + */ + private static String baseName(String name) { + final int slash = name.lastIndexOf('/'); + return slash < 0 ? name : name.substring(slash + 1); + } + + /** + * Computes the padding after an entry: tar content is stored in whole blocks. + * + * @param size The entry size in bytes. + * @return The number of padding bytes up to the next block boundary. + */ + private static long padding(long size) { + final long remainder = size % TAR_BLOCK; + return remainder == 0 ? 0 : TAR_BLOCK - remainder; + } + + /** + * Consumes and discards an exact number of bytes from the stream. + * + * @param in The stream to read from. + * @param bytes The number of bytes to discard. + * @throws IOException Thrown if the stream ends before that many bytes were read. + */ + private static void skip(InputStream in, long bytes) throws IOException { + long remaining = bytes; + final byte[] buffer = new byte[8192]; + while (remaining > 0) { + final int read = in.read(buffer, 0, (int) Math.min(buffer.length, remaining)); + if (read < 0) { + throw new IOException("truncated tar entry"); + } + remaining -= read; + } + } + + /** + * Wraps the tar stream so exactly one entry's bytes are readable. + * + * @param in The tar stream, positioned at the entry's first byte. + * @param size The entry size in bytes. + * @return A stream reporting end of stream after that many bytes, and failing if the + * tar stream ends first. Never {@code null}; closing it leaves {@code in} + * open and positioned after the entry content. + */ + private static InputStream boundedStream(InputStream in, long size) { + return new InputStream() { + private long remaining = size; + + @Override + public int read() throws IOException { + if (remaining <= 0) { + return -1; + } + final int b = in.read(); + if (b < 0) { + throw new IOException("truncated tar entry"); + } + remaining--; + return b; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + if (remaining <= 0) { + return -1; + } + final int read = in.read(buffer, offset, (int) Math.min(length, remaining)); + if (read < 0) { + throw new IOException("truncated tar entry"); + } + remaining -= read; + return read; + } + }; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java new file mode 100644 index 0000000000..a6694ddc58 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.util.List; + +import opennlp.tools.util.Span; + +/** + * One morpheme from lattice segmentation: the {@link Span} it covers in the original + * text, its surface form, and the feature columns its dictionary entry carries. + * + *

The features are the entry's columns exactly as listed in the dictionary, since + * different dictionaries carry different schemas: part of speech first by convention, + * then dictionary-specific columns such as conjugation, base form, or reading. A + * morpheme produced by unknown-word handling has the unknown entry's features and is + * marked as such.

+ * + * @param span The location of the morpheme in the original text. Must not be + * {@code null}. + * @param surface The covered text. Must not be {@code null} or empty. + * @param features The dictionary feature columns. Must not be {@code null}. + * @param unknown Whether the morpheme came from unknown-word handling rather than a + * lexicon entry. + * + * @since 3.0.0 + */ +public record Morpheme(Span span, String surface, List features, + boolean unknown) { + + /** + * Validates the morpheme. + * + * @throws IllegalArgumentException Thrown if {@code span}, {@code surface}, or + * {@code features} is {@code null}, or {@code surface} is empty. + */ + public Morpheme { + if (span == null) { + throw new IllegalArgumentException("span must not be null"); + } + if (surface == null || surface.isEmpty()) { + throw new IllegalArgumentException("surface must not be null or empty"); + } + if (features == null) { + throw new IllegalArgumentException("features must not be null"); + } + features = List.copyOf(features); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java new file mode 100644 index 0000000000..060a0a53d2 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; + +/** + * Frequency-driven segmentation for Chinese and similar scripts: a Viterbi search that + * maximizes the summed log-probability of the words in a user-supplied frequency + * lexicon, with unlisted characters falling back to single-character words. This is the + * unigram model behind common Chinese segmenters; it carries no connection costs, so it + * is lighter than the {@link LatticeTokenizer} and fits lexicons that list only words + * and counts. + * + *

The lexicon format is one entry per line: the word, its count, and optionally a + * tag, separated by whitespace. The lexicon file is user-supplied; no lexicon data is + * bundled. Every reported span is in original text coordinates.

+ * + *

Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +public class UnigramSegmenter implements Tokenizer { + + /** The log-probability charged to a character the lexicon does not know. */ + private final double unknownLogProbability; + + private final WordTrie trie; + + /** A minimal trie over words with their log-probabilities. */ + private static final class WordTrie { + private final Map children = new HashMap<>(); + private double logProbability = Double.NaN; + } + + private UnigramSegmenter(WordTrie trie, double unknownLogProbability) { + this.trie = trie; + this.unknownLogProbability = unknownLogProbability; + } + + /** + * Loads a frequency lexicon encoded in UTF-8. + * + * @param lexicon The lexicon file. Must not be {@code null}. + * @return The segmenter. Never {@code null}. + * @throws IOException Thrown if reading fails or the lexicon is empty or malformed. + * @throws IllegalArgumentException Thrown if {@code lexicon} is {@code null}. + */ + public static UnigramSegmenter load(Path lexicon) throws IOException { + return load(lexicon, StandardCharsets.UTF_8); + } + + /** + * Loads a frequency lexicon. + * + * @param lexicon The lexicon file: one word, its count, and an optional tag per + * line. Must not be {@code null}. + * @param charset The lexicon encoding. Must not be {@code null}. + * @return The segmenter. Never {@code null}. + * @throws IOException Thrown if reading fails or the lexicon is empty or malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static UnigramSegmenter load(Path lexicon, Charset charset) throws IOException { + if (lexicon == null) { + throw new IllegalArgumentException("lexicon must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); + } + try (InputStream in = Files.newInputStream(lexicon)) { + return load(in, charset); + } + } + + /** + * Loads a frequency lexicon from a stream. + * + * @param lexiconStream The lexicon content. Must not be {@code null}. Not closed. + * @param charset The lexicon encoding. Must not be {@code null}. + * @return The segmenter. Never {@code null}. + * @throws IOException Thrown if reading fails or the lexicon is empty or malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) + throws IOException { + if (lexiconStream == null) { + throw new IllegalArgumentException("lexiconStream must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); + } + final Map counts = new HashMap<>(); + long total = 0; + final String content = new String(lexiconStream.readAllBytes(), charset); + int lineStart = 0; + int lineNumber = 0; + for (int i = 0; i <= content.length(); i++) { + if (i < content.length() && content.charAt(i) != '\n') { + continue; + } + lineNumber++; + final String line = StringUtil.trimUnicodeWhitespace(content.substring(lineStart, i)); + lineStart = i + 1; + if (line.isEmpty()) { + continue; + } + final int wordEnd = whitespaceIndex(line); + if (wordEnd < 0) { + throw new IOException("lexicon line " + lineNumber + " has no count"); + } + final String word = line.substring(0, wordEnd); + int countStart = wordEnd; + while (countStart < line.length() && StringUtil.isWhitespace(line.charAt(countStart))) { + countStart++; + } + int countEnd = countStart; + while (countEnd < line.length() && !StringUtil.isWhitespace(line.charAt(countEnd))) { + countEnd++; + } + final long count; + try { + count = Long.parseLong(line.substring(countStart, countEnd)); + } catch (NumberFormatException e) { + throw new IOException("malformed count at lexicon line " + lineNumber, e); + } + if (count <= 0) { + throw new IOException("count must be positive at lexicon line " + lineNumber); + } + counts.merge(word, count, Long::sum); + total += count; + } + if (counts.isEmpty()) { + throw new IOException("the lexicon lists no words"); + } + + final WordTrie root = new WordTrie(); + final double logTotal = Math.log(total); + for (final Map.Entry entry : counts.entrySet()) { + WordTrie node = root; + final String word = entry.getKey(); + for (int c = 0; c < word.length(); c++) { + node = node.children.computeIfAbsent(word.charAt(c), + key -> new WordTrie()); + } + node.logProbability = Math.log(entry.getValue()) - logTotal; + } + // Charge an unlisted character half of one count out of the total, which makes it + // rarer than any listed word: every listed count is at least one. + final double unknown = Math.log(0.5) - logTotal; + return new UnigramSegmenter(root, unknown); + } + + /** + * {@inheritDoc} + * + *

Reports the segmented surfaces, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + @Override + public String[] tokenize(String text) { + final Span[] spans = tokenizePos(text); + final String[] tokens = new String[spans.length]; + for (int i = 0; i < tokens.length; i++) { + tokens[i] = text.substring(spans[i].getStart(), spans[i].getEnd()); + } + return tokens; + } + + /** + * {@inheritDoc} + * + *

Reports the segmented spans in original text coordinates, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + @Override + public Span[] tokenizePos(String text) { + if (text == null) { + throw new IllegalArgumentException("text must not be null"); + } + final List spans = new ArrayList<>(); + int start = 0; + while (start < text.length()) { + if (StringUtil.isWhitespace(text.charAt(start))) { + start++; + continue; + } + int end = start; + while (end < text.length() && !StringUtil.isWhitespace(text.charAt(end))) { + end++; + } + decode(text, start, end, spans); + start = end; + } + return spans.toArray(new Span[0]); + } + + /** + * Viterbi over word log-probabilities within one whitespace-free stretch. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param spans Receives the best path's spans, in text order and in original text + * coordinates. + */ + private void decode(String text, int from, int to, List spans) { + final int length = to - from; + final double[] best = new double[length + 1]; + final int[] previous = new int[length + 1]; + for (int i = 1; i <= length; i++) { + best[i] = Double.NEGATIVE_INFINITY; + } + for (int i = 0; i < length; i++) { + if (best[i] == Double.NEGATIVE_INFINITY) { + continue; + } + // A single-character step at the unknown log-probability keeps every position + // reachable even where no lexicon word matches. The step advances one code + // point, never one code unit, so an unknown supplementary character is stepped + // over whole and no span boundary can land between its surrogate halves. + final int width = Character.charCount(text.codePointAt(from + i)); + final double fallback = best[i] + unknownLogProbability; + if (i + width <= length && fallback > best[i + width]) { + best[i + width] = fallback; + previous[i + width] = i; + } + WordTrie node = trie; + for (int j = from + i; j < to; j++) { + node = node.children.get(text.charAt(j)); + if (node == null) { + break; + } + if (!Double.isNaN(node.logProbability)) { + final int end = j - from + 1; + final double score = best[i] + node.logProbability; + if (score > best[end]) { + best[end] = score; + previous[end] = i; + } + } + } + } + final List reversed = new ArrayList<>(); + for (int end = length; end > 0; end = previous[end]) { + reversed.add(new Span(from + previous[end], from + end)); + } + for (int i = reversed.size() - 1; i >= 0; i--) { + spans.add(reversed.get(i)); + } + } + + /** + * Finds the first whitespace character in a lexicon line. + * + * @param text The line to scan. + * @return The index of the first whitespace character, or {@code -1} when the line + * contains none. + */ + private static int whitespaceIndex(String text) { + for (int i = 0; i < text.length(); i++) { + if (StringUtil.isWhitespace(text.charAt(i))) { + return i; + } + } + return -1; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java new file mode 100644 index 0000000000..56a3ea8710 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -0,0 +1,721 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.tools.util.Span; + +/** + * Tests the lattice segmenter against a project-authored miniature dictionary; no + * external dictionary data is involved. + * + *

Source strings are written as Unicode escapes to keep this file ASCII-only; the + * class works over the same miniature Japanese dictionary as the sibling usage + * example, whose javadoc spells out each fixture word.

+ */ +public class LatticeTokenizerTest { + + private static final String LEXICON_CSV = "lexicon.csv"; + private static final String MATRIX_DEF = "matrix.def"; + private static final String CHAR_DEF = "char.def"; + private static final String UNK_DEF = "unk.def"; + + /** A one by one connection matrix charging cost zero, for single-context fixtures. */ + private static final String UNIT_MATRIX = "1 1\n0 0 0\n"; + + /** + * The {@code char.def} line defining the DEFAULT category: it does not invoke + * unknown-word handling beside a lexicon match, it groups a whole run into one + * candidate, and it offers no fixed-length candidates. + */ + private static final String DEFAULT_CATEGORY_LINE = "DEFAULT 0 1 0"; + + /** The {@code unk.def} template line for the DEFAULT category. */ + private static final String DEFAULT_UNKNOWN_TEMPLATE = "DEFAULT,0,0,10000,symbol,unknown"; + + @TempDir + static Path directory; + + private static LatticeTokenizer tokenizer; + + @BeforeAll + static void loadDictionary() throws IOException { + write(LEXICON_CSV, String.join("\n", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", + "")); + write(MATRIX_DEF, UNIT_MATRIX); + write(CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "LATIN 1 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "0x0041..0x005A LATIN", + "0x0061..0x007A LATIN", + "")); + write(UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, + "LATIN,0,0,4000,noun,foreign", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "")); + tokenizer = new LatticeTokenizer(MecabDictionary.load(directory)); + } + + /** Writes one UTF-8 dictionary file into the shared dictionary directory. */ + private static void write(String name, String content) throws IOException { + write(directory, name, content); + } + + /** Writes one UTF-8 dictionary file into a test-supplied directory. */ + private static void write(Path target, String name, String content) throws IOException { + Files.write(target.resolve(name), content.getBytes(StandardCharsets.UTF_8)); + } + + @Test + void testLatticePrefersTheCheaperSegmentation() { + // Tokyo plus the metropolis suffix must beat the competing reading east plus Kyoto. + final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + } + + @Test + void testMorphemesCarryDictionaryFeatures() { + final List morphemes = + tokenizer.analyze("\u6771\u4EAC\u90FD\u306B\u884C\u304F"); + Assertions.assertEquals(4, morphemes.size()); + Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); + Assertions.assertEquals(List.of("particle", "case"), morphemes.get(2).features()); + Assertions.assertFalse(morphemes.get(0).unknown()); + } + + @Test + void testUnknownLatinRunGroupsIntoOneMorpheme() { + final List morphemes = tokenizer.analyze("ABC\u306B\u884C\u304F"); + Assertions.assertEquals(3, morphemes.size()); + Assertions.assertEquals("ABC", morphemes.get(0).surface()); + Assertions.assertTrue(morphemes.get(0).unknown()); + Assertions.assertEquals(List.of("noun", "foreign"), morphemes.get(0).features()); + } + + @Test + void testUnknownKanjiPreferOneMorphemeOverTwo() { + final List morphemes = tokenizer.analyze("\u5CE0\u9053\u306B\u884C\u304F"); + Assertions.assertEquals(3, morphemes.size()); + Assertions.assertEquals("\u5CE0\u9053", morphemes.get(0).surface()); + Assertions.assertTrue(morphemes.get(0).unknown()); + } + + /** + * Verifies that an unknown-word candidate never spans a character category boundary. + * An unlisted kanji directly followed by a Latin letter must be analyzed as two + * morphemes of their own categories, never as one KANJI morpheme whose surface glues + * the kanji to the letter. + */ + @Test + void testUnknownCandidatesNeverSpanCategoryBoundaries() { + final String text = "\u5CE0a"; + Assertions.assertArrayEquals(new String[] {"\u5CE0", "a"}, tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1), new Span(1, 2)}, + tokenizer.tokenizePos(text)); + final List morphemes = tokenizer.analyze(text); + Assertions.assertEquals(List.of("noun", "unknown"), morphemes.get(0).features()); + Assertions.assertEquals(List.of("noun", "foreign"), morphemes.get(1).features()); + } + + /** + * Verifies that bounding unknown-word candidates by the category run does not under + * generate inside the run: a two-kanji unlisted run followed by a Latin letter still + * offers the length-two KANJI candidate, which wins over two single-kanji morphemes. + */ + @Test + void testUnknownRunStillOffersWithinCategoryLengths() { + final String text = "\u5CE0\u9053a"; + Assertions.assertArrayEquals(new String[] {"\u5CE0\u9053", "a"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 2), new Span(2, 3)}, + tokenizer.tokenizePos(text)); + } + + @Test + void testWhitespaceSeparatesAndIsNeverAMorpheme() { + final String text = "\u6771\u4EAC \u306B \u884C\u304F"; + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(3, 4), new Span(5, 7)}, + tokenizer.tokenizePos(text)); + Assertions.assertEquals(0, tokenizer.analyze(" ").size()); + Assertions.assertEquals(0, tokenizer.analyze("").size()); + } + + /** + * Verifies that empty input yields empty results from every view of the tokenizer. + */ + @Test + void testEmptyInputYieldsEmptyResults() { + Assertions.assertArrayEquals(new String[0], tokenizer.tokenize("")); + Assertions.assertArrayEquals(new Span[0], tokenizer.tokenizePos("")); + } + + /** + * Verifies single-character input for a listed surface and for an unlisted kanji: + * both come back as exactly one morpheme covering {@code [0, 1)}, and only the + * unlisted one is marked unknown. + */ + @Test + void testSingleCharacterInput() { + Assertions.assertArrayEquals(new String[] {"\u306B"}, tokenizer.tokenize("\u306B")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("\u306B")); + Assertions.assertFalse(tokenizer.analyze("\u306B").get(0).unknown()); + + final List unknown = tokenizer.analyze("\u5CE0"); + Assertions.assertEquals(1, unknown.size()); + Assertions.assertEquals("\u5CE0", unknown.get(0).surface()); + Assertions.assertEquals(new Span(0, 1), unknown.get(0).span()); + Assertions.assertTrue(unknown.get(0).unknown()); + } + + /** + * Verifies input made entirely of characters absent from both the lexicon and the + * {@code char.def} mappings: they fall into the DEFAULT category, whose grouping + * setting joins the whole same-category run into one unknown morpheme carrying the + * DEFAULT template's features. + */ + @Test + void testEntirelyUnknownInputGroupsIntoOneDefaultMorpheme() { + final List morphemes = tokenizer.analyze("\u2460\u2461\u2462"); + Assertions.assertEquals(1, morphemes.size()); + Assertions.assertEquals("\u2460\u2461\u2462", morphemes.get(0).surface()); + Assertions.assertEquals(new Span(0, 3), morphemes.get(0).span()); + Assertions.assertTrue(morphemes.get(0).unknown()); + Assertions.assertEquals(List.of("symbol", "unknown"), morphemes.get(0).features()); + } + + /** + * Verifies a mixed run of known and unknown text: the lexicon words around an + * unmapped character are kept intact, the unmapped character becomes its own + * unknown morpheme, and every span stays in original text coordinates. + */ + @Test + void testMixedKnownAndUnknownRuns() { + final String text = "\u6771\u4EAC\u2460\u306B\u884C\u304F"; + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u2460", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + final List morphemes = tokenizer.analyze(text); + Assertions.assertFalse(morphemes.get(0).unknown()); + Assertions.assertTrue(morphemes.get(1).unknown()); + Assertions.assertFalse(morphemes.get(2).unknown()); + } + + /** + * Verifies that spans keep original text coordinates when the interesting content + * does not start at position zero because of leading whitespace. + */ + @Test + void testSpansStayOriginalAfterLeadingWhitespace() { + final String text = " \u6771\u4EAC\u90FD\u306B\u884C\u304F"; + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(2, 4), new Span(4, 5), new Span(5, 6), new Span(6, 8)}, + tokenizer.tokenizePos(text)); + } + + /** + * Verifies that a lexicon row with fewer than the four mandatory columns is + * rejected at load time. + */ + @Test + void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { + // The rest of the dictionary is well formed, so the short row is what load rejects. + writeUnitMatrixDictionary(broken); + write(broken, LEXICON_CSV, "\u6771,0,0\n"); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a non-numeric cost column in a lexicon row is rejected at load + * time. + */ + @Test + void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { + // The rest of the dictionary is well formed, so the cost column is what load rejects. + writeUnitMatrixDictionary(broken); + write(broken, LEXICON_CSV, "\u6771,0,0,abc,noun\n"); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a {@code matrix.def} data line with the wrong number of fields is + * rejected at load time. + */ + @Test + void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, "1 1\n0 0\n"); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a {@code char.def} code point mapping without a category name is + * rejected at load time. + */ + @Test + void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) + throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, UNIT_MATRIX); + write(broken, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\n0x4E00..0x9FFF\n"); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies the fail-loud path when a loadable dictionary cannot cover the input: the + * {@code unk.def} has no DEFAULT template, so a character with neither a lexicon + * entry nor a category template stops segmentation with an exception instead of + * being dropped silently. + */ + @Test + void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) + throws IOException { + write(partial, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(partial, MATRIX_DEF, UNIT_MATRIX); + write(partial, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n"); + write(partial, UNK_DEF, "KANJI,0,0,8000,noun\n"); + final LatticeTokenizer limited = + new LatticeTokenizer(MecabDictionary.load(partial)); + Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("\u2460")); + } + + /** + * Verifies that a directory holding a lexicon but none of the definition files is + * rejected at load time, naming the first file that is missing. + */ + @Test + void testMissingDefinitionFileFailsLoud(@TempDir Path broken) throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("required dictionary file is missing: " + + broken.resolve(MATRIX_DEF), e.getMessage()); + } + + /** + * Verifies that a {@code char.def} without the mandatory DEFAULT category is rejected + * at load time rather than leaving unmapped code points without a fallback. + */ + @Test + void testCharDefWithoutDefaultCategoryFailsLoud(@TempDir Path broken) throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, UNIT_MATRIX); + write(broken, CHAR_DEF, "KANJI 0 0 2\n0x4E00..0x9FFF KANJI\n"); + write(broken, UNK_DEF, "KANJI,0,0,8000,noun\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("char.def defines no DEFAULT category: " + + broken.resolve(CHAR_DEF), e.getMessage()); + } + + /** + * Verifies that a directory with the definition files but no lexicon entry at all is + * rejected at load time, since no text could be segmented against it. + */ + @Test + void testDictionaryWithoutLexiconEntriesFailsLoud(@TempDir Path empty) throws IOException { + writeUnitMatrixDictionary(empty); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(empty)); + Assertions.assertEquals("no lexicon entries found under " + empty, e.getMessage()); + } + + /** + * Verifies that an empty {@code matrix.def} is reported as such instead of as a + * malformed header with nothing to show. + */ + @Test + void testEmptyMatrixDefFailsLoud(@TempDir Path broken) throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, ""); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("empty matrix.def under " + broken, e.getMessage()); + } + + /** + * Verifies the {@code char.def} fail-loud paths that a malformed line can take: a + * descending code point range, a code point outside the Unicode range, a code point + * field that is not hexadecimal, and a category line missing its length column. + * + * @param charDef The {@code char.def} content under test. + * @param broken The directory the fixture dictionary is written into. + * @throws IOException Thrown if writing the fixture fails. + */ + @ParameterizedTest(name = "[{index}] char.def {0}") + @ValueSource(strings = { + DEFAULT_CATEGORY_LINE + "\n0x0110..0x0100 LATIN\n", + DEFAULT_CATEGORY_LINE + "\n0x110000 LATIN\n", + DEFAULT_CATEGORY_LINE + "\n0xZZ LATIN\n", + "DEFAULT 0 1\n"}) + void testMalformedCharDefFailsLoud(String charDef, @TempDir Path broken) + throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, UNIT_MATRIX); + write(broken, CHAR_DEF, charDef); + write(broken, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Writes a miniature dictionary whose {@code char.def} maps a supplementary plane + * range, the shape a UniDic-style distribution uses for the CJK extension blocks. + * + * @param target The directory to write the dictionary files into. Must not be + * {@code null} and must exist. + * @throws IOException Thrown if writing any of the files fails. + */ + private static void writeSupplementaryDictionary(Path target) throws IOException { + write(target, LEXICON_CSV, "\u6771,0,0,6000,noun,common\n"); + write(target, MATRIX_DEF, UNIT_MATRIX); + write(target, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "KANJI 0 0 2", + "LATIN 1 1 0", + "", + "0x4E00..0x9FFF KANJI", + "0x20000..0x2A6DF KANJI", + "0x0061..0x007A LATIN", + "")); + write(target, UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, + "KANJI,0,0,8000,noun,unknown", + "LATIN,0,0,4000,noun,foreign", + "")); + } + + /** + * Verifies that a {@code char.def} range above U+FFFF is honored rather than + * discarded: a supplementary plane ideograph inside the mapped range takes the + * category the range names, while a supplementary code point outside every mapped + * range still falls back to DEFAULT. + */ + @Test + void testSupplementaryCharDefRangeIsHonored(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final MecabDictionary dictionary = MecabDictionary.load(supplementary); + // U+20BB7 is a CJK extension B ideograph inside the mapped range. + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x20BB7).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x6771).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x2460).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x2A6E0).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf('a').name()); + } + + /** + * Verifies that a supplementary plane ideograph is analyzed as the single character + * it is: one morpheme whose span covers both code units and which carries the + * features of the category its {@code char.def} range names, never one morpheme per + * surrogate. The second case shows the category's length templates count characters, + * not code units, so a run of two supplementary ideographs is still reachable by the + * length-two template. + */ + @Test + void testSupplementaryIdeographIsOneMorpheme(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final LatticeTokenizer supplementaryTokenizer = + new LatticeTokenizer(MecabDictionary.load(supplementary)); + // U+20BB7 written as its surrogate pair, per this file's ASCII-only convention. + final String text = "\uD842\uDFB7"; + final List morphemes = supplementaryTokenizer.analyze(text); + Assertions.assertEquals(1, morphemes.size()); + Assertions.assertEquals(text, morphemes.get(0).surface()); + Assertions.assertEquals(new Span(0, 2), morphemes.get(0).span()); + Assertions.assertEquals(List.of("noun", "unknown"), morphemes.get(0).features()); + + final List pair = supplementaryTokenizer.analyze(text + text); + Assertions.assertEquals(1, pair.size()); + Assertions.assertEquals(new Span(0, 4), pair.get(0).span()); + Assertions.assertEquals(List.of("noun", "unknown"), pair.get(0).features()); + } + + /** + * Verifies that a supplementary plane ideograph does not absorb neighbouring text of + * another category: the ideograph and an unmapped symbol beside it stay two + * morphemes, each span covering whole characters. + */ + @Test + void testSupplementaryIdeographDoesNotAbsorbItsNeighbour(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final LatticeTokenizer supplementaryTokenizer = + new LatticeTokenizer(MecabDictionary.load(supplementary)); + final String text = "\uD842\uDFB7\u2460"; + Assertions.assertArrayEquals(new String[] {"\uD842\uDFB7", "\u2460"}, + supplementaryTokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 2), new Span(2, 3)}, + supplementaryTokenizer.tokenizePos(text)); + } + + /** + * Writes every dictionary file except the lexicon, so a test can supply a lexicon of + * its own against a one by one connection matrix. + * + * @param target The directory to write the dictionary files into. Must not be + * {@code null} and must exist. + * @throws IOException Thrown if writing any of the files fails. + */ + private static void writeUnitMatrixDictionary(Path target) throws IOException { + write(target, MATRIX_DEF, UNIT_MATRIX); + write(target, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\n"); + write(target, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + } + + /** + * Verifies that a lexicon row whose right context id is outside the + * {@code matrix.def} dimensions is rejected at load time, naming the file, the line, + * and the offending id, rather than reaching the cost matrix with an out of range + * index during segmentation. + */ + @Test + void testRightContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) + throws IOException { + writeUnitMatrixDictionary(mismatched); + write(mismatched, LEXICON_CSV, "\u6771,0,5,3000,noun\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(mismatched)); + Assertions.assertEquals("malformed entry at " + mismatched.resolve(LEXICON_CSV) + + " line 1: right context id 5 is outside the matrix.def dimensions 1 1", + e.getMessage()); + } + + /** + * Verifies that a lexicon row whose left context id is outside the {@code matrix.def} + * dimensions is rejected at load time, naming the file, the line, and the offending + * id. + */ + @Test + void testLeftContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) + throws IOException { + writeUnitMatrixDictionary(mismatched); + write(mismatched, LEXICON_CSV, "\u6771,0,0,3000,noun\n\u90FD,7,0,3000,noun\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(mismatched)); + Assertions.assertEquals("malformed entry at " + mismatched.resolve(LEXICON_CSV) + + " line 2: left context id 7 is outside the matrix.def dimensions 1 1", + e.getMessage()); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LatticeTokenizer(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionary.load(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionary.load(null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionary.load(directory, null)); + Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.analyze(null)); + Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.tokenize(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> tokenizer.tokenizePos(null)); + } + + /** + * Verifies the {@link Morpheme} contract every segmentation result is built from: a + * {@code null} span, a {@code null} or empty surface, and {@code null} features are + * all rejected, and the feature list is copied so a later change to the caller's list + * cannot be seen through the morpheme. + */ + @Test + void testMorphemeRejectsInvalidArguments() { + final Span span = new Span(0, 1); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(null, "東", List.of("noun"), false)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(span, null, List.of("noun"), false)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(span, "", List.of("noun"), false)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(span, "東", null, false)); + + final List features = new ArrayList<>(List.of("noun")); + final Morpheme morpheme = new Morpheme(span, "東", features, false); + features.add("proper"); + Assertions.assertEquals(List.of("noun"), morpheme.features()); + } + + /** + * Verifies the supplementary range table's interval cutting and precedence: a later + * {@code char.def} mapping strictly inside an earlier one wins exactly on its own + * stretch, and the earlier category resumes after it, so the cut produces three + * intervals from two overlapping ranges. + */ + @Test + void testLaterSupplementaryMappingWinsInsideAnEarlierRange(@TempDir Path overlapped) + throws IOException { + write(overlapped, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(overlapped, MATRIX_DEF, UNIT_MATRIX); + write(overlapped, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "KANJI 0 0 2", + "LATIN 1 1 0", + "", + "0x20000..0x2FFFF KANJI", + "0x24000..0x25000 LATIN", + "")); + write(overlapped, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + + final MecabDictionary dictionary = MecabDictionary.load(overlapped); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x20000).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x23FFF).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x24000).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x25000).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x25001).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x2FFFF).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x30000).name()); + } + + /** + * Verifies a {@code char.def} range straddling the BMP boundary: the part up to + * U+FFFF lands in the directly indexed table and the rest in the range table, and + * both halves answer the same category with no gap at the seam. + */ + @Test + void testCharDefRangeStraddlingTheBmpBoundary(@TempDir Path straddling) + throws IOException { + write(straddling, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(straddling, MATRIX_DEF, UNIT_MATRIX); + write(straddling, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "LATIN 1 1 0", + "", + "0xFF00..0x10040 LATIN", + "")); + write(straddling, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + + final MecabDictionary dictionary = MecabDictionary.load(straddling); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0xFF00).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0xFFFF).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x10000).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x10040).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x10041).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0xFEFF).name()); + } + + /** + * Verifies that a {@code char.def} mapping to a category its category section never + * defined fails at load, naming the code point and the ghost category, instead of + * silently falling back to DEFAULT at lookup time. + */ + @Test + void testMappingToUndefinedCategoryFailsLoud(@TempDir Path ghost) throws IOException { + write(ghost, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(ghost, MATRIX_DEF, UNIT_MATRIX); + write(ghost, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "", + "0x0100..0x0110 GHOST", + "")); + write(ghost, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(ghost)); + Assertions.assertEquals("char.def maps U+0100 to the undefined category GHOST", + e.getMessage()); + } + + /** + * Verifies that a connection cost outside the 16-bit range the binary matrix format + * defines is rejected at load instead of being truncated by the narrowing cast into + * a silently different cost. + */ + @Test + void testMatrixCostOutsideShortRangeFailsLoud(@TempDir Path broken) throws IOException { + writeUnitMatrixDictionary(broken); + write(broken, MATRIX_DEF, "1 1\n0 0 40000\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("malformed matrix.def line 2: connection cost 40000 is" + + " outside the 16-bit range the format defines", e.getMessage()); + } + + /** + * Verifies that {@code matrix.def} dimensions whose product exceeds the addressable + * array size fail loud at the header instead of overflowing the int multiplication + * into a negative or wrapped allocation size. + */ + @Test + void testMatrixDimensionProductBeyondIntRangeFailsLoud(@TempDir Path broken) + throws IOException { + writeUnitMatrixDictionary(broken); + write(broken, MATRIX_DEF, "70000 70000\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("matrix.def dimensions 70000 x 70000 overflow the" + + " addressable connection matrix", e.getMessage()); + } + + /** + * Verifies that a {@code matrix.def} data row naming context ids outside the + * declared dimensions is rejected at load with the offending line and ids. + */ + @Test + void testMatrixRowContextIdsOutsideDimensionsFailLoud(@TempDir Path broken) + throws IOException { + writeUnitMatrixDictionary(broken); + write(broken, MATRIX_DEF, "1 1\n2 0 5\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("malformed matrix.def line 2: context ids 2 0 are outside" + + " the declared dimensions 1 1", e.getMessage()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java new file mode 100644 index 0000000000..0fc58585de --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.Span; + +/** + * Demonstrates the intended end-to-end usage of this package with miniature, + * project-authored data: a mecab-format dictionary archive is installed with + * {@link MecabDictionaryInstaller}, loaded as a {@link MecabDictionary}, and segmented + * with a {@link LatticeTokenizer}; a plain frequency lexicon is loaded and segmented + * with a {@link UnigramSegmenter}. Everything is written to a temporary directory by + * the test itself; no external dictionary or lexicon data and no network access are + * involved. + * + *

Source strings are written as Unicode escapes to keep this file ASCII-only. The + * Japanese fixture words are Tokyo (U+6771 U+4EAC), Kyoto (U+4EAC U+90FD), east + * (U+6771), the metropolis suffix (U+90FD), the case particle ni (U+306B), and the + * verb iku, to go (U+884C U+304F); the Chinese fixture words are wo, I (U+6211), + * laidao, arrive (U+6765 U+5230), Beijing (U+5317 U+4EAC), and Tiananmen + * (U+5929 U+5B89 U+95E8).

+ */ +public class LatticeUsageExampleTest { + + /** + * Walks the full mecab-format flow: package a miniature Japanese dictionary as a + * {@code tar.gz} archive, install it from a file URI, load it, and tokenize. The + * segmentation must pick the cheaper path (Tokyo plus the metropolis suffix) over + * the competing reading (east plus Kyoto), the spans must be in original text + * coordinates, and the morphemes must carry the dictionary's feature columns. + */ + @Test + void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) + throws IOException { + // A minimal but complete dictionary: one lexicon file plus the three definition + // files every mecab-format distribution contains, wrapped like a release archive. + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { + {"mini-dict-0.1/lexicon.csv", String.join("\n", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", + "")}, + {"mini-dict-0.1/matrix.def", "1 1\n0 0 0\n"}, + {"mini-dict-0.1/char.def", String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "")}, + {"mini-dict-0.1/unk.def", String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "")}, + {"mini-dict-0.1/README", "not a dictionary payload file"}}); + final Path archiveFile = work.resolve("mini-dict-0.1.tar.gz"); + Files.write(archiveFile, archive); + + // Install: fetch the archive from the user-chosen location and unpack the payload. + final Path dictionaryDirectory = work.resolve("dictionary"); + final int extracted = + MecabDictionaryInstaller.install(archiveFile.toUri(), dictionaryDirectory); + Assertions.assertEquals(4, extracted); + + // Load and tokenize; both views must agree and stay in original coordinates. + final LatticeTokenizer tokenizer = + new LatticeTokenizer(MecabDictionary.load(dictionaryDirectory)); + final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + + // The analyze view adds the dictionary's feature columns to every morpheme. + final List morphemes = tokenizer.analyze(text); + Assertions.assertEquals(4, morphemes.size()); + Assertions.assertEquals("\u6771\u4EAC", morphemes.get(0).surface()); + Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); + Assertions.assertFalse(morphemes.get(0).unknown()); + } + + /** + * Walks the frequency-lexicon flow: write a miniature word-count lexicon to a file, + * load it, and segment. The segmentation must recover the listed multi-character + * words with spans in original text coordinates. + */ + @Test + void testLoadAndSegmentWithAFrequencyLexicon(@TempDir Path work) throws IOException { + // One word, its count, and an optional tag per line, whitespace separated. + final Path lexicon = work.resolve("words.txt"); + Files.write(lexicon, String.join("\n", + "\u6211 5000 r", + "\u6765\u5230 2000 v", + "\u5317\u4EAC 3000 ns", + "\u5929\u5B89\u95E8 1200 ns", + "").getBytes(StandardCharsets.UTF_8)); + + final UnigramSegmenter segmenter = UnigramSegmenter.load(lexicon); + final String text = "\u6211\u6765\u5230\u5317\u4EAC\u5929\u5B89\u95E8"; + Assertions.assertArrayEquals( + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC", "\u5929\u5B89\u95E8"}, + segmenter.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 8)}, + segmenter.tokenizePos(text)); + } + + /** + * Walks the non-UTF-8 flow that widely used Japanese distributions require: the same + * miniature dictionary is written to disk encoded in EUC-JP and loaded through the + * charset-taking overload. The segmentation must match the UTF-8 run exactly, which + * shows the encoding is a property of loading, not of tokenization. + */ + @Test + void testLoadAnEucJpEncodedDictionary(@TempDir Path work) throws IOException { + final Charset eucJp = Charset.forName("EUC-JP"); + Files.write(work.resolve("lexicon.csv"), String.join("\n", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", + "").getBytes(eucJp)); + Files.write(work.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(eucJp)); + Files.write(work.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "").getBytes(eucJp)); + Files.write(work.resolve("unk.def"), String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "").getBytes(eucJp)); + + final LatticeTokenizer tokenizer = + new LatticeTokenizer(MecabDictionary.load(work, eucJp)); + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize("\u6771\u4EAC\u90FD\u306B\u884C\u304F")); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java new file mode 100644 index 0000000000..fb5c6a701f --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests the installer against project-authored, in-memory archives; no external + * dictionary data and no network access are involved. + */ +public class MecabDictionaryInstallerTest { + + @Test + void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target) + throws IOException { + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { + {"dict-1.0/lexicon.csv", "cat,0,0,100,noun\n"}, + {"dict-1.0/matrix.def", "1 1\n0 0 0\n"}, + {"dict-1.0/char.def", "DEFAULT 0 1 0\n"}, + {"dict-1.0/unk.def", "DEFAULT,0,0,10000,unknown\n"}, + {"dict-1.0/README", "not a dictionary file"}, + {"dict-1.0/dicrc", "config"}}); + + final int extracted = MecabDictionaryInstaller.extract( + new ByteArrayInputStream(archive), target); + + Assertions.assertEquals(5, extracted); + Assertions.assertTrue(Files.exists(target.resolve("lexicon.csv"))); + Assertions.assertTrue(Files.exists(target.resolve("matrix.def"))); + Assertions.assertTrue(Files.exists(target.resolve("char.def"))); + Assertions.assertTrue(Files.exists(target.resolve("unk.def"))); + Assertions.assertTrue(Files.exists(target.resolve("dicrc"))); + Assertions.assertTrue(Files.notExists(target.resolve("README"))); + Assertions.assertEquals("cat,0,0,100,noun\n", + Files.readString(target.resolve("lexicon.csv"))); + } + + @Test + void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target) + throws IOException { + final Path archiveFile = source.resolve("dict.tar.gz"); + Files.write(archiveFile, TarGzArchives.gzippedTar(new String[][] { + {"d/words.csv", "cat,0,0,100,noun\n"}, + {"d/matrix.def", "1 1\n0 0 0\n"}})); + + final int extracted = + MecabDictionaryInstaller.install(archiveFile.toUri(), target); + + Assertions.assertEquals(2, extracted); + Assertions.assertTrue(Files.exists(target.resolve("words.csv"))); + } + + @Test + void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target) + throws IOException { + final byte[] archive = + TarGzArchives.gzippedTar(new String[][] {{"readme.txt", "nothing here"}}); + Assertions.assertThrows(IOException.class, () -> MecabDictionaryInstaller.extract( + new ByteArrayInputStream(archive), target)); + } + + @Test + void testInvalidArguments(@TempDir Path target) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.install(null, target)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.install(target.toUri(), null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.extract(null, target)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.extract( + new ByteArrayInputStream(new byte[0]), null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java new file mode 100644 index 0000000000..896115aba2 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPOutputStream; + +/** + * Builds miniature, project-authored gzip-compressed ustar archives in memory for the + * tests of this package; no external archive data is involved. + */ +final class TarGzArchives { + + /** The tar block size; headers, content, and padding are all whole blocks. */ + private static final int BLOCK = 512; + + /** The header field offsets and lengths this builder writes, in bytes. */ + private static final int NAME_LENGTH = 100; + private static final int MODE_OFFSET = 100; + private static final int SIZE_OFFSET = 124; + private static final int SIZE_LENGTH = 12; + private static final int CHECKSUM_OFFSET = 148; + private static final int CHECKSUM_LENGTH = 8; + private static final int TYPE_OFFSET = 156; + + /** The type flag of a regular file entry. */ + private static final char REGULAR_FILE = '0'; + + private TarGzArchives() { + } + + /** + * Builds a gzip-compressed tar archive from name and content pairs, the layout a + * dictionary distribution ships in. + * + * @param entries The entries as {@code {name, content}} pairs. Must not be + * {@code null}. + * @return The compressed archive bytes. Never {@code null}. + * @throws IOException Thrown if writing to the in-memory streams fails. + */ + static byte[] gzippedTar(String[][] entries) throws IOException { + final ByteArrayOutputStream tar = new ByteArrayOutputStream(); + for (final String[] entry : entries) { + tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); + } + // Two zero blocks end a tar archive. + tar.write(new byte[2 * BLOCK]); + final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { + gzip.write(tar.toByteArray()); + } + return compressed.toByteArray(); + } + + /** + * Appends one ustar file entry to a growing tar image: a 512-byte header block + * followed by the content padded to a block boundary. + * + * @param tar The tar image under construction. Must not be {@code null}. + * @param name The entry name including any directory prefix. Must not be + * {@code null}, empty, or longer than the 100-byte header name field. + * @param content The entry content bytes. Must not be {@code null}. + * @throws IOException Thrown if writing to the in-memory stream fails. + * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. + */ + private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) + throws IOException { + final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + if (nameBytes.length == 0 || nameBytes.length > NAME_LENGTH) { + throw new IllegalArgumentException( + "entry name must be 1.." + NAME_LENGTH + " bytes, got " + nameBytes.length); + } + final byte[] header = new byte[BLOCK]; + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(mode, 0, header, MODE_OFFSET, mode.length); + // Both numeric fields hold octal digits followed by one terminator byte. + final byte[] size = String.format("%0" + (SIZE_LENGTH - 1) + "o", content.length) + .getBytes(StandardCharsets.US_ASCII); + System.arraycopy(size, 0, header, SIZE_OFFSET, size.length); + header[TYPE_OFFSET] = REGULAR_FILE; + // The checksum is computed with its own field read as spaces. + for (int i = CHECKSUM_OFFSET; i < CHECKSUM_OFFSET + CHECKSUM_LENGTH; i++) { + header[i] = ' '; + } + int checksum = 0; + for (final byte b : header) { + checksum += b & 0xFF; + } + final byte[] checksumText = String.format("%0" + (CHECKSUM_LENGTH - 2) + "o", checksum) + .getBytes(StandardCharsets.US_ASCII); + System.arraycopy(checksumText, 0, header, CHECKSUM_OFFSET, checksumText.length); + header[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 2] = 0; + header[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 1] = ' '; + tar.write(header); + tar.write(content); + final int padding = (BLOCK - content.length % BLOCK) % BLOCK; + tar.write(new byte[padding]); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java new file mode 100644 index 0000000000..487c1c6ad3 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.tools.util.Span; + +/** + * Tests the frequency-driven segmenter against a project-authored miniature lexicon; + * no external lexicon data is involved. + * + *

Source strings are written as Unicode escapes to keep this file ASCII-only; the + * class works over the same miniature Chinese frequency lexicon as the sibling usage + * example, whose javadoc spells out each fixture word.

+ */ +public class UnigramSegmenterTest { + + private static final String LEXICON = String.join("\n", + "\u6211 5000 r", + "\u6765\u5230 2000 v", + "\u5317\u4EAC 3000 ns", + "\u6E05\u534E\u5927\u5B66 800 nt", + "\u6E05\u534E 400 ns", + "\u534E\u5927 100 ns", + "\u5927\u5B66 1500 n", + "\u7684 9000 uj", + ""); + + private static UnigramSegmenter segmenter; + + @BeforeAll + static void loadLexicon() throws IOException { + segmenter = UnigramSegmenter.load( + new ByteArrayInputStream(LEXICON.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + } + + @Test + void testPrefersWholeWordsOverFragments() { + Assertions.assertArrayEquals( + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC", "\u6E05\u534E\u5927\u5B66"}, + segmenter.tokenize("\u6211\u6765\u5230\u5317\u4EAC\u6E05\u534E\u5927\u5B66")); + } + + @Test + void testSpansStayInOriginalCoordinates() { + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 9)}, + segmenter.tokenizePos("\u6211\u6765\u5230\u5317\u4EAC\u6E05\u534E\u5927\u5B66")); + } + + @Test + void testUnknownCharactersFallBackToSingles() { + Assertions.assertArrayEquals( + new String[] {"\u6211", "\u7231", "\u5317\u4EAC"}, + segmenter.tokenize("\u6211\u7231\u5317\u4EAC")); + } + + @Test + void testWhitespaceSeparates() { + Assertions.assertArrayEquals( + new String[] {"\u5317\u4EAC", "\u5927\u5B66"}, + segmenter.tokenize("\u5317\u4EAC \u5927\u5B66")); + Assertions.assertEquals(0, segmenter.tokenizePos(" ").length); + } + + /** + * Verifies that empty input yields empty results from both views of the segmenter. + */ + @Test + void testEmptyInputYieldsEmptyResults() { + Assertions.assertArrayEquals(new String[0], segmenter.tokenize("")); + Assertions.assertArrayEquals(new Span[0], segmenter.tokenizePos("")); + } + + /** + * Verifies single-character input for a listed word and for a character the + * lexicon does not know: both come back as exactly one token covering + * {@code [0, 1)}. + */ + @Test + void testSingleCharacterInput() { + Assertions.assertArrayEquals(new String[] {"\u6211"}, segmenter.tokenize("\u6211")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("\u6211")); + Assertions.assertArrayEquals(new String[] {"\u7231"}, segmenter.tokenize("\u7231")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("\u7231")); + } + + /** + * Verifies input made entirely of characters absent from the lexicon: every + * character becomes its own single-character token, since only the unknown + * fallback is available. + */ + @Test + void testEntirelyUnknownInputFallsBackToSingleCharacters() { + Assertions.assertArrayEquals( + new String[] {"x", "y", "z"}, + segmenter.tokenize("xyz")); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 2), new Span(2, 3)}, + segmenter.tokenizePos("xyz")); + } + + /** + * Verifies a mixed run of known and unknown text inside one whitespace-free + * stretch: the unknown character becomes a single token while the listed words + * around it, including the longest listed compound, stay intact. + */ + @Test + void testMixedKnownAndUnknownRuns() { + Assertions.assertArrayEquals( + new String[] {"\u6211", "\u7231", "\u6E05\u534E\u5927\u5B66"}, + segmenter.tokenize("\u6211\u7231\u6E05\u534E\u5927\u5B66")); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 2), new Span(2, 6)}, + segmenter.tokenizePos("\u6211\u7231\u6E05\u534E\u5927\u5B66")); + } + + /** + * Verifies that spans keep original text coordinates when the content does not + * start at position zero because of leading whitespace. + */ + @Test + void testSpansStayOriginalAfterLeadingWhitespace() { + final String text = " \u6211\u6765\u5230\u5317\u4EAC"; + Assertions.assertArrayEquals( + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC"}, + segmenter.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(2, 3), new Span(3, 5), new Span(5, 7)}, + segmenter.tokenizePos(text)); + } + + @ParameterizedTest(name = "lexicon content \"{0}\"") + @ValueSource(strings = {"word\n", "word abc\n", "word 0\n", "\n\n"}) + void testMalformedLexiconsFailLoud(String lexicon) { + Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( + new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((Path) null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((Path) null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load(Path.of("words.txt"), null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((InputStream) null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load(new ByteArrayInputStream(new byte[0]), null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> segmenter.tokenize(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> segmenter.tokenizePos(null)); + } + + /** + * Verifies that the unknown-character fallback advances one code point, never one + * code unit: a supplementary character absent from the lexicon comes back as one + * span over both of its surrogate halves, and no span boundary ever lands between + * them. + */ + @Test + void testUnknownSupplementaryCharacterIsNeverSplit() { + // U+20BB7, a CJK extension B ideograph, written as its surrogate pair + final String text = "\uD842\uDFB7\uD842\uDFB7"; + final Span[] spans = segmenter.tokenizePos(text); + for (final Span span : spans) { + Assertions.assertEquals(0, span.getStart() % 2, + "span must start on a code point boundary: " + span); + Assertions.assertEquals(0, span.getEnd() % 2, + "span must end on a code point boundary: " + span); + } + int covered = 0; + for (final Span span : spans) { + covered += span.length(); + } + Assertions.assertEquals(text.length(), covered); + } + + /** + * Pins Unicode-whitespace trimming of lexicon lines: a leading ideographic space + * (U+3000), common in hand-edited CJK text files, is stripped like ASCII whitespace, + * so the entry loads rather than failing as a malformed count. + */ + @Test + void testLeadingIdeographicSpaceIsTrimmed() throws IOException { + // U+3000 ideographic space, then the fixture word U+6211 and its count + final String lexicon = "\u3000\u6211 5000 r\n"; + final UnigramSegmenter loaded = UnigramSegmenter.load( + new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + Assertions.assertArrayEquals(new String[] {"\u6211"}, loaded.tokenize("\u6211")); + } +} diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index cd1d8a2ddf..6f74b6a601 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -538,5 +538,30 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { +
+ Lattice tokenization for CJK + + Languages written without spaces need a dictionary-backed segmenter. + LatticeTokenizer scores paths over a MeCab-format dictionary and + emits the cheapest segmentation with spans in original text coordinates. + UnigramSegmenter does the same from a plain frequency lexicon. + Install a dictionary archive with MecabDictionaryInstaller, load it + as a MecabDictionary, and tokenize. + LatticeUsageExampleTest asserts the install-load-tokenize and + lexicon flows shown here. + + morphemes = tokenizer.analyze(text); + +UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt")); +String[] words = segmenter.tokenize(text);]]> + + +