XSD 1.1 store-time document validation + Saxon catalog resolver

.github/scripts/prepare-governance-context.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Pure git plumbing for schema governance — everything else (pairing,
+# version comparison, the GitHub annotations, failing the build) happens in
+# a single Saxon XSLT 2.0 transform (schema/governance.xsl) driven by
+# `mvn xml:transform@schema-governance`. This script's only job is to put
+# the git state that transform needs onto disk as plain files/XML, so the
+# stylesheet never has to shell out itself.
+set -euo pipefail
+cd "$(git rev-parse --show-toplevel)"
+
+WORKSPACE="${1:?workspace required}"
+OUT="${2:-${WORKSPACE}/target/governance}"
+mkdir -p "${OUT}/base"
+
+if [[ -n "${GITHUB_BASE_REF:-}" ]]; then
+  git fetch --depth=1 origin "${GITHUB_BASE_REF}" 2>/dev/null || true
+  BASE="$(git merge-base HEAD "origin/${GITHUB_BASE_REF}")"
+elif [[ -n "${GITHUB_EVENT_BEFORE:-}" && "${GITHUB_EVENT_BEFORE}" != "0000000000000000000000000000000000000000" ]]; then
+  BASE="${GITHUB_EVENT_BEFORE}"
+else
+  BASE="$(git rev-parse HEAD~1 2>/dev/null || git rev-parse HEAD)"
+fi
+
+# Every tracked schema/*.xsd's content as it existed at BASE, one file per
+# schema named after its basename. A missing file at $OUT/base/<name> means
+# "didn't exist at BASE" (new schema) — governance.xsl checks for that with
+# doc-available() rather than this script trying to distinguish "new file"
+# from "tool failure" itself.
+git ls-tree -r --name-only HEAD -- schema | grep '\.xsd$' | while read -r f; do
+  out="${OUT}/base/$(basename "${f}")"
+  git show "${BASE}:${f}" > "${out}" 2>/dev/null || rm -f "${out}"
+done
+
+# One path per line; governance.xsl reads this with unparsed-text() + tokenize(),
+# so no XML-escaping of path characters is needed anywhere in this pipeline.
+git diff --name-only "${BASE}" -- \
+  schema \
+  exist-distribution/src/main/config \
+  exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml \
+  exist-core/src/main/resources/org/exist/util/mime-types.xml \
+  exist-core/src/main/java/org/exist/util/SchemaVersion.java \
+  > "${OUT}/changed.txt" 2>/dev/null || true
+
+cat > "${OUT}/context.xml" <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<g:ctx xmlns:g="http://exist-db.org/ns/schema-governance"
+       workspace="${WORKSPACE}"
+       base-dir="${OUT}/base"
+       base-ref="${BASE}"
+       changed-file="${OUT}/changed.txt"/>
+EOF
+
+echo "Governance context: ${OUT}/context.xml (base ${BASE})"

.github/workflows/ci-schema-checks.yml

@@ -0,0 +1,60 @@
+name: Schema checks
+
+on:
+  pull_request:
+    paths:
+      - 'schema/**'
+      - 'exist-distribution/src/main/config/**'
+      - 'exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml'
+      - 'exist-core/src/main/resources/org/exist/util/mime-types.xml'
+      - 'exist-core/src/main/java/org/exist/util/SchemaVersion.java'
+  push:
+    branches: [develop]
+    paths:
+      - 'schema/**'
+      - 'exist-distribution/src/main/config/**'
+      - 'exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml'
+      - 'exist-core/src/main/resources/org/exist/util/mime-types.xml'
+      - 'exist-core/src/main/java/org/exist/util/SchemaVersion.java'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  MAVEN_OPTS: -DtrimStackTrace=false
+  DEV_JDK: '21'
+
+jobs:
+  schema:
+    name: Native XSD checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: ${{ env.DEV_JDK }}
+
+      - uses: ./.github/actions/maven-cache
+
+      - name: Validate canonical templates against XSD
+        run: mvn -V -B --no-transfer-progress validate -Ddependency-check.skip=true -Ddocker=false
+
+      - name: Prepare governance context
+        env:
+          GITHUB_BASE_REF: ${{ github.base_ref }}
+          GITHUB_EVENT_BEFORE: ${{ github.event.before }}
+        run: |
+          chmod +x .github/scripts/prepare-governance-context.sh
+          .github/scripts/prepare-governance-context.sh "${{ github.workspace }}"
+
+      - name: Run schema governance (XSLT 2.0 / Saxon)
+        run: mvn -N -B --no-transfer-progress xml:transform@schema-governance -Ddependency-check.skip=true -Ddocker=false
+
+      - name: Verify SchemaVersion.java matches XSD versions
+        run: mvn -B --no-transfer-progress test -pl exist-core -Dtest=org.exist.util.SchemaVersionSyncTest -Ddependency-check.skip=true -Ddocker=false

AGENTS.md

@@ -144,6 +144,30 @@ ANTLR generates `XQueryParser.java`, `XQueryLexer.java`, `XQueryTreeParser.java`
 | `org.exist.dom.persistent` | Persistent DOM implementation |
 | `org.exist.dom.memtree` | In-memory DOM (for constructed nodes) |
 
+### Native config schemas (`schema/`)
+
+eXist-db's own config-file XSDs (`conf.xsd`, `collection.xconf.xsd`, `descriptor.xsd`,
+`controller-config.xsd`, `mime-types.xsd`, plus `users.xsd`/`server.xsd`/`security-manager.xsd`/
+`expath-pkg.xsd` and its extensions) live in [`schema/`](schema/) at the repo root, and are shipped
+in every distribution layout as `$EXIST_HOME/schema/` — a sibling of `etc/`, `bin/`, `lib/` (tarball,
+zip, Docker image, and the IzPack installer all include it; see `exist-distribution`/`exist-docker`/
+`exist-installer`). External tools (eXide, IDE plugins) can resolve a config file's grammar from
+this fixed location instead of vendoring their own copy.
+
+- Each XSD's `xs:schema/@version` is an independent semver line — see [`schema/README.md`](schema/README.md)
+  for the versioning policy (CI enforces a version bump on any semantic schema edit, via
+  `mvn -N xml:transform@schema-governance`, see [`schema/governance.xsl`](schema/governance.xsl)).
+- `org.exist.util.SchemaVersion`'s version constants are generated at build time from the XSDs
+  themselves (`generate-sources` phase, see `exist-core/pom.xml`'s `schema-version-codegen`
+  execution and [`schema/generate-schema-version.xsl`](schema/generate-schema-version.xsl)) — never
+  hand-edit `SchemaVersion`'s constants; bump the XSD's `xs:schema/@version` instead and the
+  constant follows automatically on the next build.
+- The 5 canonical instances (the files `pom.xml`'s `validate-canonical-instances` execution
+  validates on every `mvn validate`) are the only ones checked for drift; the ~39 test/sample
+  fixture copies scattered across module test resources (e.g. `extensions/*/src/test/resources*/conf.xml`)
+  are intentionally hand-trimmed per-module subsets, not literal copies — don't try to regenerate
+  them from canonical.
+
 ### Adding a new `fn:` function
 
 1. Create the class in `org.exist.xquery.functions.fn` extending `BasicFunction`

exist-core/pom.xml

@@ -301,7 +301,6 @@
         <dependency>
             <groupId>org.exist-db.thirdparty.xerces</groupId>
             <artifactId>xercesImpl</artifactId>
-            <version>2.12.2</version>
             <classifier>jdk14-xml-schema-1.1</classifier>
             <exclusions>
                 <exclusion> <!-- conflicts with Java 17's javax.xml module -->
@@ -348,7 +347,6 @@
         <dependency>
             <groupId>org.xmlresolver</groupId>
             <artifactId>xmlresolver</artifactId>
-            <version>${xmlresolver.version}</version>
             <exclusions>
                 <exclusion> <!-- conflicts with Java 17's javax.xml module -->
                     <groupId>xml-apis</groupId>
@@ -360,7 +358,6 @@
         <dependency>
             <groupId>org.xmlresolver</groupId>
             <artifactId>xmlresolver</artifactId>
-            <version>${xmlresolver.version}</version>
             <classifier>data</classifier>
             <scope>runtime</scope>
             <exclusions>
@@ -376,13 +373,11 @@
         <dependency>
             <groupId>org.exist-db.thirdparty.org.eclipse.wst.xml</groupId>
             <artifactId>xpath2</artifactId>
-            <version>1.2.0</version>
             <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>edu.princeton.cup</groupId>
             <artifactId>java-cup</artifactId>
-            <version>10k</version>
             <scope>runtime</scope>
         </dependency>
 
@@ -1087,6 +1082,64 @@ The BaseX Team. The original license statement is also included below.]]></pream
                 </executions>
             </plugin>
 
+            <!--
+                Generates org/exist/util/GeneratedSchemaVersions.java from each canonical XSD's
+                xs:schema/@version (see ../schema/generate-schema-version.xsl), runs on every
+                normal build. Unlike ../schema/governance.xsl (phase=none, CI-invoked-only), this
+                means SchemaVersion.java's version constants can never silently drift from
+                schema/*.xsd.
+            -->
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>xml-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>schema-version-codegen</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>transform</goal>
+                        </goals>
+                        <configuration>
+                            <forceCreation>true</forceCreation>
+                            <transformationSets>
+                                <transformationSet>
+                                    <dir>${project.basedir}/../schema</dir>
+                                    <includes>
+                                        <include>GeneratedSchemaVersions.xml</include>
+                                    </includes>
+                                    <stylesheet>${project.basedir}/../schema/generate-schema-version.xsl</stylesheet>
+                                    <outputDir>${project.build.directory}/generated-sources/schema-version/org/exist/util</outputDir>
+                                    <fileMappers>
+                                        <fileMapper implementation="org.codehaus.plexus.components.io.filemappers.FileExtensionMapper">
+                                            <targetExtension>.java</targetExtension>
+                                        </fileMapper>
+                                    </fileMappers>
+                                </transformationSet>
+                            </transformationSets>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <version>3.6.0</version>
+                <executions>
+                    <execution>
+                        <id>add-schema-version-source</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${project.build.directory}/generated-sources/schema-version</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>

exist-core/src/main/java/org/exist/Namespaces.java

@@ -39,6 +39,7 @@ public interface Namespaces {
 	String SCHEMA_NS = XMLConstants.W3C_XML_SCHEMA_NS_URI;
     String SCHEMA_DATATYPES_NS = "http://www.w3.org/2001/XMLSchema-datatypes";
 	String SCHEMA_INSTANCE_NS = XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI;
+    String XSD_1_1_NS = "http://www.w3.org/XML/XMLSchema/v1.1";
 
     // Move this here from Function.BUILTIN_FUNCTION_NS? /ljo
 	String XPATH_FUNCTIONS_NS = "http://www.w3.org/2005/xpath-functions";	

exist-core/src/main/java/org/exist/collections/CollectionConfiguration.java

@@ -40,6 +40,7 @@
 import org.exist.storage.IndexSpec;
 import org.exist.util.DatabaseConfigurationException;
 import org.exist.util.ParametersExtractor;
+import org.exist.util.SchemaVersion;
 import org.exist.util.XMLReaderObjectFactory;
 import org.exist.xmldb.XmldbURI;
 import org.w3c.dom.Document;
@@ -129,6 +130,8 @@ protected void read(final DBBroker broker, final Document doc, final boolean che
                     "' in configuration document. Got '" + root.getNamespaceURI() + "'", checkOnly);
             return;
         }
+        SchemaVersion.logDocumentVersion(LOG, root, SchemaVersion.COLLECTION_XCONF,
+                "collection.xconf" + (docName != null ? " (" + docName + ")" : ""));
         final NodeList childNodes = root.getChildNodes();
         for (int i = 0; i < childNodes.getLength(); i++) {
             Node node = childNodes.item(i);

exist-core/src/main/java/org/exist/collections/IndexInfo.java

@@ -96,13 +96,30 @@ void setReader(final XMLReader reader, final EntityResolver entityResolver) thro
         if(entityResolver != null) {
             reader.setEntityResolver(entityResolver);
         }
-        final LexicalHandler lexicalHandler = docTriggers == null ? indexer : docTriggers;
-        final ContentHandler contentHandler = docTriggers == null ? indexer : docTriggers;
-        reader.setProperty(Namespaces.SAX_LEXICAL_HANDLER, lexicalHandler);
-        reader.setContentHandler(contentHandler);
+        reader.setProperty(Namespaces.SAX_LEXICAL_HANDLER, getLexicalHandler());
+        reader.setContentHandler(getContentHandler());
         reader.setErrorHandler(indexer);
     }
 
+    /**
+     * The same content handler {@link #setReader(XMLReader, EntityResolver)} wires onto an
+     * {@link XMLReader} -- exposed so callers that validate via a {@link javax.xml.validation.ValidatorHandler}
+     * instead of an {@code XMLReader} (which has no equivalent {@code setReader}-style helper) can feed
+     * the same indexing/trigger pipeline through a SAX-driven validation pass.
+     */
+    ContentHandler getContentHandler() {
+        return docTriggers == null ? indexer : docTriggers;
+    }
+
+    /**
+     * The same lexical handler {@link #setReader(XMLReader, EntityResolver)} wires onto an
+     * {@link XMLReader} -- see {@link #getContentHandler()} for why this is also exposed
+     * separately.
+     */
+    LexicalHandler getLexicalHandler() {
+        return docTriggers == null ? indexer : docTriggers;
+    }
+
     void setDOMStreamer(final DOMStreamer streamer) {
         this.streamer = streamer;
         if (docTriggers == null) {

exist-core/src/main/java/org/exist/collections/MutableCollection.java

@@ -117,6 +117,15 @@ public class MutableCollection implements Collection {
     private final Permission permissions;
     @Deprecated private CollectionMetadata collectionMetadata = null;
 
+    /**
+     * Discards all cached XSD 1.1 schema-by-namespace resolutions -- called alongside
+     * {@code Jaxp.clearXsd11DetectionCache()} by {@code validation:clear-grammar-cache()}
+     * so one admin action clears both XSD-1.1-detection caches, not just the schemaLocation-hint one.
+     */
+    public static void clearXsd11SchemaByNamespaceCache() {
+        Xsd11ValidationHelper.clearSchemaCache();
+    }
+
     /**
      * Constructs a Collection Object (not yet persisted)
      *
@@ -1125,9 +1134,8 @@ public void storeDocument(final Txn transaction, final DBBroker broker, final Xm
             // Store XML Document
 
             final BiConsumer2E<XMLReader, IndexInfo, SAXException, EXistException> validatorFn = (xmlReader1, validateIndexInfo) -> {
-                validateIndexInfo.setReader(xmlReader1, null);
                 try {
-                      xmlReader1.parse(source);
+                    Xsd11ValidationHelper.parseOrValidateXmlSource(broker, xmlReader1, validateIndexInfo, source);
                 } catch(final SAXException e) {
                     throw new SAXException("The XML parser reported a problem: " + e.getMessage(), e);
                 } catch(final IOException e) {
@@ -1137,8 +1145,7 @@ public void storeDocument(final Txn transaction, final DBBroker broker, final Xm
 
             final BiConsumer2E<XMLReader, IndexInfo, SAXException, EXistException> parserFn = (xmlReader1, storeIndexInfo) -> {
                 try {
-                    storeIndexInfo.setReader(xmlReader1, null);
-                    xmlReader1.parse(source);
+                    Xsd11ValidationHelper.parseOrValidateXmlSource(broker, xmlReader1, storeIndexInfo, source);
                 } catch(final IOException e) {
                     throw new EXistException(e);
                 }