diff --git a/.github/workflows/ci-benchmarks.yml b/.github/workflows/ci-benchmarks.yml new file mode 100644 index 00000000000..71e8e49d631 --- /dev/null +++ b/.github/workflows/ci-benchmarks.yml @@ -0,0 +1,100 @@ +name: JMH Benchmarks +on: + workflow_dispatch: + schedule: + # Weekly, Sunday 04:00 UTC - distinct from ci-container.yml (daily 06:00) + # and prethink.yml (Monday 06:00) so runners aren't all queued together. + - cron: '0 4 * * 0' + +permissions: + contents: read # default; the benchmarks job declares write explicitly + +# Only one publish to gh-pages at a time; a manual dispatch overlapping the +# scheduled run would otherwise race pushing the benchmark-data commit. +concurrency: + group: jmh-benchmarks-gh-pages + cancel-in-progress: false + +env: + DEV_JDK: '21' + # Kept small enough that both benchmark suites fit in the job timeout on a + # shared GitHub-hosted runner; not intended to produce publishable numbers, + # only a consistent trend line. See exist-core-jmh/README.md and + # exist-indexes-jmh/README.md for slower, more precise local invocations. + JMH_ARGS: '-wi 2 -i 5 -f 1' + +jobs: + benchmarks: + name: Run JMH benchmarks + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: write # github-action-benchmark pushes the data series to gh-pages + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: ${{ env.DEV_JDK }} + + - uses: ./.github/actions/maven-cache + + - name: Build exist-core-jmh + run: > + mvn -B install -pl exist-core-jmh -am -DskipTests + -Ddependency-check.skip=true -Ddocker=false + + - name: Run exist-core-jmh benchmarks + run: > + java -jar exist-core-jmh/target/exist-core-jmh-*-benchmarks.jar + -rf json -rff exist-core-jmh/target/jmh-result.json + ${{ env.JMH_ARGS }} + + - name: Track exist-core-jmh results on gh-pages + if: github.repository == 'eXist-db/exist' + uses: benchmark-action/github-action-benchmark@v1 + with: + name: 'exist-core-jmh' + tool: 'jmh' + output-file-path: exist-core-jmh/target/jmh-result.json + benchmark-data-dir-path: dev/bench/core + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Report-only: flag regressions, never fail the run. + alert-threshold: '150%' + comment-on-alert: true + fail-on-alert: false + summary-always: true + + - name: Build exist-indexes-jmh + run: > + mvn -B install -pl exist-indexes-jmh -am -DskipTests + -Ddependency-check.skip=true -Ddocker=false + + - name: Run exist-indexes-jmh benchmarks + run: > + mvn -B exec:exec -pl exist-indexes-jmh + -Dbenchmark.args="-rf json -rff target/jmh-result.json ${{ env.JMH_ARGS }}" + + - name: Track exist-indexes-jmh results on gh-pages + if: github.repository == 'eXist-db/exist' + uses: benchmark-action/github-action-benchmark@v1 + with: + name: 'exist-indexes-jmh' + tool: 'jmh' + output-file-path: exist-indexes-jmh/target/jmh-result.json + benchmark-data-dir-path: dev/bench/indexes + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + alert-threshold: '150%' + comment-on-alert: true + fail-on-alert: false + summary-always: true + + - name: Save Maven cache + if: github.event_name == 'schedule' + uses: actions/cache/save@v6 + with: + path: ~/.m2 + key: maven-${{ github.ref_name }}-${{ hashFiles('**/pom.xml') }} diff --git a/exist-core-jmh/README.md b/exist-core-jmh/README.md new file mode 100644 index 00000000000..aa84568db07 --- /dev/null +++ b/exist-core-jmh/README.md @@ -0,0 +1,48 @@ +# exist-core-jmh + +JMH micro-benchmarks for `exist-core` (and the Lucene index extension, which several benchmarks here also exercise). Each class exists to make a specific optimization or regression claim defensible with numbers instead of prose — see the class-level Javadoc for the issue/PR each one is a companion to. + +## What's benchmarked + +| Class | Package | Measures | +|---|---|---| +| `AxisBenchmark` | `org.exist.dom.persistent` | Sibling/non-sibling axis evaluation over three corpus shapes ([#2697](https://github.com/eXist-db/exist/issues/2697)) | +| `PrecedingAxisBenchmark` | `org.exist.xquery` | `preceding::*` position-dependence on a flat 50,000-element document ([#2129](https://github.com/eXist-db/exist/issues/2129)) | +| `ArrowOperatorBenchmark` | `org.exist.xquery` | Overhead of `=>` vs. the equivalent direct call | +| `TypeSubTypeOfBenchmark` | `org.exist.xquery.value` | `Type#subTypeOf(int, int)`, the hot path behind every atomic comparison ([#6322](https://github.com/eXist-db/exist/issues/6322)) | +| `StringJoinBenchmark` | `org.exist.xquery.utils` | Java `String` join strategies | +| `URIUtilsBenchmark` | `org.exist.xquery.utils` | URI utility helpers | +| `LockTableBenchmark` | `org.exist.storage.lock` | Lock table contention | +| `ReindexBenchmark` | `org.exist.storage` | End-to-end `xmldb:reindex()` fast path ([#572](https://github.com/eXist-db/exist/issues/572)) | +| `LucenePhraseQueryBenchmark` | `org.exist.indexing.lucene` | Query-time Lucene phrase lookup over many small documents ([#2812](https://github.com/eXist-db/exist/issues/2812)) | +| `ReindexDeleteStrategyBenchmark` | `org.exist.indexing.lucene` | Reindex-time Lucene delete strategies for mixed document shapes | +| `UtilExpandHighlightingBenchmark` | `org.exist.indexing.lucene` | `util:expand` match-highlighting cost vs. highlighting off, single-hit and batch ([#5738](https://github.com/eXist-db/exist/issues/5738), [#6387](https://github.com/eXist-db/exist/issues/6387)) | + +## Running + +Build the module first — this also installs a fresh `exist-core` into the local Maven repo so the benchmark picks up your branch's code: + +```bash +JAVA_HOME=/path/to/java-21 \ + mvn install -pl exist-core-jmh -am -DskipTests \ + -Ddependency-check.skip=true -Ddocker=false +``` + +The `package` phase shades an uber-jar with `org.openjdk.jmh.Main` as its entry point (`target/exist-core-jmh--benchmarks.jar`). Run it directly: + +```bash +java -jar exist-core-jmh/target/exist-core-jmh-*-benchmarks.jar AxisBenchmark -wi 3 -i 5 -f 1 +``` + +Useful variants: + +- `... -rf json -rff target/jmh-result.json` — machine-readable output (this is what CI feeds to the gh-pages dashboard) +- `... -prof gc` — GC profile +- `... TypeSubTypeOfBenchmark.identical` — filter by regex +- No class filter runs every `@Benchmark` in the jar + +`ArrowOperatorBenchmark` needs the *unshaded* classes plus runtime classpath instead of the shaded jar — the shade transformer trips a log4j2 caller-class assertion when booting a `BrokerPool`. See that class's Javadoc for the exact invocation. + +## Continuous tracking + +[`.github/workflows/ci-benchmarks.yml`](../.github/workflows/ci-benchmarks.yml) runs the full suite weekly (and on manual dispatch) with reduced iteration counts (`-wi 2 -i 5 -f 1` — a trend signal, not a publishable number) and publishes JSON results via [`benchmark-action/github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) to the `gh-pages` branch, under `dev/bench/core`. Once GitHub Pages is enabled for this repository, the dashboard is served at `https://exist-db.github.io/exist/dev/bench/core/`. diff --git a/exist-core-jmh/src/main/java/org/exist/indexing/lucene/UtilExpandHighlightingBenchmark.java b/exist-core-jmh/src/main/java/org/exist/indexing/lucene/UtilExpandHighlightingBenchmark.java new file mode 100644 index 00000000000..f99b2415434 --- /dev/null +++ b/exist-core-jmh/src/main/java/org/exist/indexing/lucene/UtilExpandHighlightingBenchmark.java @@ -0,0 +1,290 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.indexing.lucene; + +import org.exist.EXistException; +import org.exist.collections.Collection; +import org.exist.collections.CollectionConfigurationException; +import org.exist.collections.CollectionConfigurationManager; +import org.exist.collections.triggers.TriggerException; +import org.exist.security.PermissionDeniedException; +import org.exist.storage.BrokerPool; +import org.exist.storage.DBBroker; +import org.exist.storage.txn.TransactionManager; +import org.exist.storage.txn.Txn; +import org.exist.test.ExistEmbeddedServer; +import org.exist.util.DatabaseConfigurationException; +import org.exist.util.LockException; +import org.exist.util.MimeType; +import org.exist.util.StringInputSource; +import org.exist.xmldb.XmldbURI; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.XQuery; +import org.exist.xquery.CompiledXQuery; +import org.exist.xquery.value.Sequence; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +/** + * JMH benchmark for {@code util:expand} match-highlighting cost against Lucene full-text + * hits, migrated from the perf-ratio assertions in the {@code UtilExpandHighlightingPerformanceTest} + * that PR #6318 initially added and then removed (see #6387). + * + *

PR #6318 fixed #5738: {@code + * util:expand} with match-highlighting on was orders of magnitude slower than with highlighting + * off, because {@link LuceneMatchListener#reset(org.exist.storage.DBBroker, org.exist.dom.persistent.NodeProxy)} + * re-rewrote each query's Lucene terms on every node. The fix caches rewritten terms per Query + * identity (bounded LRU, see {@code LuceneMatchListener}) and short-circuits {@code scanMatches} + * when the configured-field exclusion (PR #3467) produces an empty term map.

+ * + *

Each shape below has a paired {@code *HighlightingOff} / {@code *HighlightingOn} benchmark + * over an identical corpus and query, mirroring {@code ArrowOperatorBenchmark}'s + * {@code arrow*}/{@code direct*} pairing: the two series plotted together on the JMH dashboard + * show both the absolute cost and its trend, without a hard-coded ratio threshold (which would be + * flaky on shared CI runners - the reason these two tests were pulled out of surefire in the + * first place).

+ * + * + * + *

Build & run (from project root)

+ * 
{@code
+ * mvn install -pl exist-core-jmh -am -DskipTests \
+ *     -Ddependency-check.skip=true -Ddocker=false
+ * java -jar exist-core-jmh/target/exist-core-jmh-7.0.0-SNAPSHOT-benchmarks.jar \
+ *     UtilExpandHighlightingBenchmark -f 1 -wi 2 -i 5
+ * }
+ * + * @see #5738 + * @see #6318 + * @see #6387 + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 3, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) +@Fork(1) +@State(Scope.Benchmark) +public class UtilExpandHighlightingBenchmark { + + private static final XmldbURI TEST_COLLECTION = XmldbURI.create("/db/bench-util-expand-highlight"); + + private static final int ENTRY_COUNT = 5000; + private static final int PARAGRAPHS_PER_ENTRY = 20; + + private static final String LUCENE_CONFIG = """ + + + + + + + + + + + """; + + private static final String COLL = "collection('" + TEST_COLLECTION + "')"; + private static final String HIGHLIGHT_OFF_OPTIONS = "'highlight-matches=none expand-xincludes=no'"; + + private ExistEmbeddedServer server; + private BrokerPool pool; + + private CompiledXQuery singleHitHighlightingOffQuery; + private CompiledXQuery singleHitHighlightingOnQuery; + private CompiledXQuery batchWildcardHighlightingOffQuery; + private CompiledXQuery batchWildcardHighlightingOnQuery; + + private int expectedWildcardHitCount; + + @Setup(Level.Trial) + public void setUp() throws EXistException, DatabaseConfigurationException, IOException, + PermissionDeniedException, CollectionConfigurationException, LockException, + SAXException, TriggerException, XPathException { + ensureExistHome(); + final Properties configProperties = new Properties(); + // BrokerPool expects a Long for this property (see BrokerPool.PROPERTY_SHUTDOWN_DELAY). + configProperties.put("wait-before-shutdown", 0L); + server = new ExistEmbeddedServer(configProperties, true, true); + server.startDb(); + pool = server.getBrokerPool(); + + storeCorpus(); + + // Half the corpus (even i) gets an 'a'-prefixed headword; matches the lemma:a* wildcard. + expectedWildcardHitCount = (ENTRY_COUNT + 1) / 2; + + try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject()))) { + final XQuery xquery = pool.getXQueryService(); + + singleHitHighlightingOffQuery = compile(xquery, broker, + "util:expand(subsequence(" + COLL + "//entry[ft:query(., 'aword42')], 1, 1), " + + HIGHLIGHT_OFF_OPTIONS + ")"); + singleHitHighlightingOnQuery = compile(xquery, broker, + "util:expand(subsequence(" + COLL + "//entry[ft:query(., 'aword42')], 1, 1))"); + batchWildcardHighlightingOffQuery = compile(xquery, broker, + "util:expand(" + COLL + "//entry[ft:query(., 'lemma:a*')], " + HIGHLIGHT_OFF_OPTIONS + ")"); + batchWildcardHighlightingOnQuery = compile(xquery, broker, + "util:expand(" + COLL + "//entry[ft:query(., 'lemma:a*')])"); + } + } + + private static CompiledXQuery compile(final XQuery xquery, final DBBroker broker, final String query) + throws XPathException, PermissionDeniedException { + final XQueryContext context = new XQueryContext(broker.getBrokerPool()); + return xquery.compile(context, query); + } + + private static void ensureExistHome() { + if (System.getProperty("exist.home") != null) { + return; + } + // Prefer the Lucene extension's own test conf (it registers the Lucene index/module and + // the 'ft' function prefix); fall back to exist-core's if run outside a full checkout. + final Path luceneHome = Path.of("extensions", "indexes", "lucene", "src", "test", "resources-filtered") + .toAbsolutePath() + .normalize(); + final Path existHome = Files.exists(luceneHome.resolve("conf.xml")) + ? luceneHome + : Path.of("exist-core", "src", "test", "resources-filtered").toAbsolutePath().normalize(); + System.setProperty("exist.home", existHome.toString()); + } + + @TearDown(Level.Trial) + public void tearDown() { + if (server != null) { + server.stopDb(); + } + } + + @Benchmark + public int expandSingleHitHighlightingOff() throws EXistException, PermissionDeniedException, XPathException, IOException { + return execute(singleHitHighlightingOffQuery, 1); + } + + @Benchmark + public int expandSingleHitHighlightingOn() throws EXistException, PermissionDeniedException, XPathException, IOException { + return execute(singleHitHighlightingOnQuery, 1); + } + + @Benchmark + public int expandBatchWildcardHighlightingOff() throws EXistException, PermissionDeniedException, XPathException, IOException { + return execute(batchWildcardHighlightingOffQuery, expectedWildcardHitCount); + } + + @Benchmark + public int expandBatchWildcardHighlightingOn() throws EXistException, PermissionDeniedException, XPathException, IOException { + return execute(batchWildcardHighlightingOnQuery, expectedWildcardHitCount); + } + + /** + * Runs the query and returns the resulting node count, throwing if it doesn't match the + * expected hit count - a "fast but wrong" guard, not a performance threshold (see class + * Javadoc: the ratio itself is read off the JMH/dashboard series, not asserted here). + */ + private int execute(final CompiledXQuery compiledQuery, final int expectedCount) + throws EXistException, PermissionDeniedException, XPathException, IOException { + try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject()))) { + final XQuery xquery = pool.getXQueryService(); + final Sequence result = xquery.execute(broker, compiledQuery, null); + final int count = result.getItemCount(); + if (count != expectedCount) { + throw new IllegalStateException("Expected " + expectedCount + " top-level results, got " + count); + } + return count; + } + } + + private void storeCorpus() throws EXistException, PermissionDeniedException, IOException, + CollectionConfigurationException, LockException, SAXException, TriggerException { + final TransactionManager transact = pool.getTransactionManager(); + try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject())); + final Txn tx = transact.beginTransaction()) { + + final Collection coll = broker.getOrCreateCollection(tx, TEST_COLLECTION); + broker.saveCollection(tx, coll); + + final CollectionConfigurationManager mgr = pool.getConfigurationManager(); + mgr.addConfiguration(tx, broker, coll, LUCENE_CONFIG); + + broker.storeDocument(tx, XmldbURI.create("dict.xml"), new StringInputSource(generateCorpus()), + MimeType.XML_TYPE, coll); + + transact.commit(tx); + } + } + + /** + * Dict/entry corpus: {@value #ENTRY_COUNT} entries, half with an 'a'-prefixed headword (the + * {@code lemma:a*} wildcard target), each padded with {@value #PARAGRAPHS_PER_ENTRY} + * paragraphs so per-entry tokenization cost is measurable - mirrors the corpus shape in the + * original (deleted) {@code UtilExpandHighlightingPerformanceTest}, minus the TEI namespace + * (dropped in the xqsuite migration as boilerplate without correctness value; irrelevant to + * the perf shape measured here). + */ + private static String generateCorpus() { + final StringBuilder doc = new StringBuilder(); + doc.append("\n"); + for (int i = 0; i < ENTRY_COUNT; i++) { + final String letter = (i % 2 == 0) ? "a" : "b"; + final String word = letter + "word" + i; + doc.append(" ") + .append("
").append(word).append("
") + .append("Definition for ").append(word).append(". "); + for (int j = 0; j < PARAGRAPHS_PER_ENTRY; j++) { + doc.append("This is paragraph ").append(j).append(" of the explanation for ") + .append(word).append(", with additional descriptive sentences ") + .append("that emulate real lexicographic content. The headword ") + .append(word).append(" appears multiple times in the body. "); + } + doc.append("\n"); + } + doc.append("\n"); + return doc.toString(); + } +} diff --git a/exist-indexes-jmh/README.md b/exist-indexes-jmh/README.md index 2861451a831..718a6224c80 100644 --- a/exist-indexes-jmh/README.md +++ b/exist-indexes-jmh/README.md @@ -158,11 +158,14 @@ exist-indexes-jmh/ `conf.xml` is generated at build time by `xml-maven-plugin` from `extensions/indexes/indexes-integration-tests/src/test/resources-filtered/conf.xml` plus the XSLT in `src/main/xslt/conf-jmh.xslt`. Mirroring the `exist-docker` pattern keeps the conf in sync with upstream rather than vendored as a literal copy. +## Continuous tracking + +[`.github/workflows/ci-benchmarks.yml`](../.github/workflows/ci-benchmarks.yml) runs this suite weekly (and on manual dispatch) with reduced iteration counts (`-wi 2 -i 5 -f 1` — a trend signal, not a publishable number) and publishes JSON results via [`benchmark-action/github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) to the `gh-pages` branch, under `dev/bench/indexes`. Once GitHub Pages is enabled for this repository, the dashboard is served at `https://exist-db.github.io/exist/dev/bench/indexes/`. + ## Roadmap Planned follow-ups (not in this PR): - Larger corpus via on-demand XMark generation (`xmlgen -f 0.1 / 1 / 10`) -- CI nightly job posting numbers to a tracking issue - `range:eq` / `range:field-eq` family extensions (`range:gt`, `range:starts-with`, etc.) if regressions turn out to be operator-specific - Diagnostics that explain *why* range, range-field, and lucene are immune to the bug ngram exhibits — the surface explanation ("their `getDependencies()` returns the right thing") is true but not actionable; the underlying difference would inform the GH-2204 fix design