diff --git a/.github/scripts/prepare-governance-context.sh b/.github/scripts/prepare-governance-context.sh new file mode 100755 index 00000000000..43b48759644 --- /dev/null +++ b/.github/scripts/prepare-governance-context.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Pure git plumbing for schema governance — everything else (pairing, +# version comparison, the GitHub annotations, failing the build) happens in +# a single Saxon XSLT 2.0 transform (schema/governance.xsl) driven by +# `mvn xml:transform@schema-governance`. This script's only job is to put +# the git state that transform needs onto disk as plain files/XML, so the +# stylesheet never has to shell out itself. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +WORKSPACE="${1:?workspace required}" +OUT="${2:-${WORKSPACE}/target/governance}" +mkdir -p "${OUT}/base" + +if [[ -n "${GITHUB_BASE_REF:-}" ]]; then + git fetch --depth=1 origin "${GITHUB_BASE_REF}" 2>/dev/null || true + BASE="$(git merge-base HEAD "origin/${GITHUB_BASE_REF}")" +elif [[ -n "${GITHUB_EVENT_BEFORE:-}" && "${GITHUB_EVENT_BEFORE}" != "0000000000000000000000000000000000000000" ]]; then + BASE="${GITHUB_EVENT_BEFORE}" +else + BASE="$(git rev-parse HEAD~1 2>/dev/null || git rev-parse HEAD)" +fi + +# Every tracked schema/*.xsd's content as it existed at BASE, one file per +# schema named after its basename. A missing file at $OUT/base/ means +# "didn't exist at BASE" (new schema) — governance.xsl checks for that with +# doc-available() rather than this script trying to distinguish "new file" +# from "tool failure" itself. +git ls-tree -r --name-only HEAD -- schema | grep '\.xsd$' | while read -r f; do + out="${OUT}/base/$(basename "${f}")" + git show "${BASE}:${f}" > "${out}" 2>/dev/null || rm -f "${out}" +done + +# One path per line; governance.xsl reads this with unparsed-text() + tokenize(), +# so no XML-escaping of path characters is needed anywhere in this pipeline. +git diff --name-only "${BASE}" -- \ + schema \ + exist-distribution/src/main/config \ + exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml \ + exist-core/src/main/resources/org/exist/util/mime-types.xml \ + exist-core/src/main/java/org/exist/util/SchemaVersion.java \ + > "${OUT}/changed.txt" 2>/dev/null || true + +cat > "${OUT}/context.xml" < + +EOF + +echo "Governance context: ${OUT}/context.xml (base ${BASE})" diff --git a/.github/workflows/ci-schema-checks.yml b/.github/workflows/ci-schema-checks.yml new file mode 100644 index 00000000000..bed3744ff75 --- /dev/null +++ b/.github/workflows/ci-schema-checks.yml @@ -0,0 +1,60 @@ +name: Schema checks + +on: + pull_request: + paths: + - 'schema/**' + - 'exist-distribution/src/main/config/**' + - 'exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml' + - 'exist-core/src/main/resources/org/exist/util/mime-types.xml' + - 'exist-core/src/main/java/org/exist/util/SchemaVersion.java' + push: + branches: [develop] + paths: + - 'schema/**' + - 'exist-distribution/src/main/config/**' + - 'exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml' + - 'exist-core/src/main/resources/org/exist/util/mime-types.xml' + - 'exist-core/src/main/java/org/exist/util/SchemaVersion.java' + workflow_dispatch: + +permissions: + contents: read + +env: + MAVEN_OPTS: -DtrimStackTrace=false + DEV_JDK: '21' + +jobs: + schema: + name: Native XSD checks + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: ${{ env.DEV_JDK }} + + - uses: ./.github/actions/maven-cache + + - name: Validate canonical templates against XSD + run: mvn -V -B --no-transfer-progress validate -Ddependency-check.skip=true -Ddocker=false + + - name: Prepare governance context + env: + GITHUB_BASE_REF: ${{ github.base_ref }} + GITHUB_EVENT_BEFORE: ${{ github.event.before }} + run: | + chmod +x .github/scripts/prepare-governance-context.sh + .github/scripts/prepare-governance-context.sh "${{ github.workspace }}" + + - name: Run schema governance (XSLT 2.0 / Saxon) + run: mvn -N -B --no-transfer-progress xml:transform@schema-governance -Ddependency-check.skip=true -Ddocker=false + + - name: Verify SchemaVersion.java matches XSD versions + run: mvn -B --no-transfer-progress test -pl exist-core -Dtest=org.exist.util.SchemaVersionSyncTest -Ddependency-check.skip=true -Ddocker=false diff --git a/AGENTS.md b/AGENTS.md index b742b4da0e5..f75437ddace 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -144,6 +144,30 @@ ANTLR generates `XQueryParser.java`, `XQueryLexer.java`, `XQueryTreeParser.java` | `org.exist.dom.persistent` | Persistent DOM implementation | | `org.exist.dom.memtree` | In-memory DOM (for constructed nodes) | +### Native config schemas (`schema/`) + +eXist-db's own config-file XSDs (`conf.xsd`, `collection.xconf.xsd`, `descriptor.xsd`, +`controller-config.xsd`, `mime-types.xsd`, plus `users.xsd`/`server.xsd`/`security-manager.xsd`/ +`expath-pkg.xsd` and its extensions) live in [`schema/`](schema/) at the repo root, and are shipped +in every distribution layout as `$EXIST_HOME/schema/` — a sibling of `etc/`, `bin/`, `lib/` (tarball, +zip, Docker image, and the IzPack installer all include it; see `exist-distribution`/`exist-docker`/ +`exist-installer`). External tools (eXide, IDE plugins) can resolve a config file's grammar from +this fixed location instead of vendoring their own copy. + +- Each XSD's `xs:schema/@version` is an independent semver line — see [`schema/README.md`](schema/README.md) + for the versioning policy (CI enforces a version bump on any semantic schema edit, via + `mvn -N xml:transform@schema-governance`, see [`schema/governance.xsl`](schema/governance.xsl)). +- `org.exist.util.SchemaVersion`'s version constants are generated at build time from the XSDs + themselves (`generate-sources` phase, see `exist-core/pom.xml`'s `schema-version-codegen` + execution and [`schema/generate-schema-version.xsl`](schema/generate-schema-version.xsl)) — never + hand-edit `SchemaVersion`'s constants; bump the XSD's `xs:schema/@version` instead and the + constant follows automatically on the next build. +- The 5 canonical instances (the files `pom.xml`'s `validate-canonical-instances` execution + validates on every `mvn validate`) are the only ones checked for drift; the ~39 test/sample + fixture copies scattered across module test resources (e.g. `extensions/*/src/test/resources*/conf.xml`) + are intentionally hand-trimmed per-module subsets, not literal copies — don't try to regenerate + them from canonical. + ### Adding a new `fn:` function 1. Create the class in `org.exist.xquery.functions.fn` extending `BasicFunction` diff --git a/exist-core/pom.xml b/exist-core/pom.xml index 94287e0e7e1..bbe7a79ac3b 100644 --- a/exist-core/pom.xml +++ b/exist-core/pom.xml @@ -301,7 +301,6 @@ org.exist-db.thirdparty.xerces xercesImpl - 2.12.2 jdk14-xml-schema-1.1 @@ -348,7 +347,6 @@ org.xmlresolver xmlresolver - ${xmlresolver.version} xml-apis @@ -360,7 +358,6 @@ org.xmlresolver xmlresolver - ${xmlresolver.version} data runtime @@ -376,13 +373,11 @@ org.exist-db.thirdparty.org.eclipse.wst.xml xpath2 - 1.2.0 runtime edu.princeton.cup java-cup - 10k runtime @@ -1087,6 +1082,64 @@ The BaseX Team. The original license statement is also included below.]]> + + + org.codehaus.mojo + xml-maven-plugin + + + schema-version-codegen + generate-sources + + transform + + + true + + + ${project.basedir}/../schema + + GeneratedSchemaVersions.xml + + ${project.basedir}/../schema/generate-schema-version.xsl + ${project.build.directory}/generated-sources/schema-version/org/exist/util + + + .java + + + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + add-schema-version-source + generate-sources + + add-source + + + + ${project.build.directory}/generated-sources/schema-version + + + + + + org.apache.maven.plugins maven-compiler-plugin diff --git a/exist-core/src/main/java/org/exist/Namespaces.java b/exist-core/src/main/java/org/exist/Namespaces.java index 593ab890857..796a5c6d333 100644 --- a/exist-core/src/main/java/org/exist/Namespaces.java +++ b/exist-core/src/main/java/org/exist/Namespaces.java @@ -39,6 +39,7 @@ public interface Namespaces { String SCHEMA_NS = XMLConstants.W3C_XML_SCHEMA_NS_URI; String SCHEMA_DATATYPES_NS = "http://www.w3.org/2001/XMLSchema-datatypes"; String SCHEMA_INSTANCE_NS = XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI; + String XSD_1_1_NS = "http://www.w3.org/XML/XMLSchema/v1.1"; // Move this here from Function.BUILTIN_FUNCTION_NS? /ljo String XPATH_FUNCTIONS_NS = "http://www.w3.org/2005/xpath-functions"; diff --git a/exist-core/src/main/java/org/exist/collections/CollectionConfiguration.java b/exist-core/src/main/java/org/exist/collections/CollectionConfiguration.java index 379eb2e6b90..ffaa06de747 100644 --- a/exist-core/src/main/java/org/exist/collections/CollectionConfiguration.java +++ b/exist-core/src/main/java/org/exist/collections/CollectionConfiguration.java @@ -40,6 +40,7 @@ import org.exist.storage.IndexSpec; import org.exist.util.DatabaseConfigurationException; import org.exist.util.ParametersExtractor; +import org.exist.util.SchemaVersion; import org.exist.util.XMLReaderObjectFactory; import org.exist.xmldb.XmldbURI; import org.w3c.dom.Document; @@ -129,6 +130,8 @@ protected void read(final DBBroker broker, final Document doc, final boolean che "' in configuration document. Got '" + root.getNamespaceURI() + "'", checkOnly); return; } + SchemaVersion.logDocumentVersion(LOG, root, SchemaVersion.COLLECTION_XCONF, + "collection.xconf" + (docName != null ? " (" + docName + ")" : "")); final NodeList childNodes = root.getChildNodes(); for (int i = 0; i < childNodes.getLength(); i++) { Node node = childNodes.item(i); diff --git a/exist-core/src/main/java/org/exist/collections/IndexInfo.java b/exist-core/src/main/java/org/exist/collections/IndexInfo.java index 0f987241024..28fbccb8d39 100644 --- a/exist-core/src/main/java/org/exist/collections/IndexInfo.java +++ b/exist-core/src/main/java/org/exist/collections/IndexInfo.java @@ -96,13 +96,30 @@ void setReader(final XMLReader reader, final EntityResolver entityResolver) thro if(entityResolver != null) { reader.setEntityResolver(entityResolver); } - final LexicalHandler lexicalHandler = docTriggers == null ? indexer : docTriggers; - final ContentHandler contentHandler = docTriggers == null ? indexer : docTriggers; - reader.setProperty(Namespaces.SAX_LEXICAL_HANDLER, lexicalHandler); - reader.setContentHandler(contentHandler); + reader.setProperty(Namespaces.SAX_LEXICAL_HANDLER, getLexicalHandler()); + reader.setContentHandler(getContentHandler()); reader.setErrorHandler(indexer); } + /** + * The same content handler {@link #setReader(XMLReader, EntityResolver)} wires onto an + * {@link XMLReader} -- exposed so callers that validate via a {@link javax.xml.validation.ValidatorHandler} + * instead of an {@code XMLReader} (which has no equivalent {@code setReader}-style helper) can feed + * the same indexing/trigger pipeline through a SAX-driven validation pass. + */ + ContentHandler getContentHandler() { + return docTriggers == null ? indexer : docTriggers; + } + + /** + * The same lexical handler {@link #setReader(XMLReader, EntityResolver)} wires onto an + * {@link XMLReader} -- see {@link #getContentHandler()} for why this is also exposed + * separately. + */ + LexicalHandler getLexicalHandler() { + return docTriggers == null ? indexer : docTriggers; + } + void setDOMStreamer(final DOMStreamer streamer) { this.streamer = streamer; if (docTriggers == null) { diff --git a/exist-core/src/main/java/org/exist/collections/MutableCollection.java b/exist-core/src/main/java/org/exist/collections/MutableCollection.java index 8010eeed46a..5e4e407469c 100644 --- a/exist-core/src/main/java/org/exist/collections/MutableCollection.java +++ b/exist-core/src/main/java/org/exist/collections/MutableCollection.java @@ -117,6 +117,15 @@ public class MutableCollection implements Collection { private final Permission permissions; @Deprecated private CollectionMetadata collectionMetadata = null; + /** + * Discards all cached XSD 1.1 schema-by-namespace resolutions -- called alongside + * {@code Jaxp.clearXsd11DetectionCache()} by {@code validation:clear-grammar-cache()} + * so one admin action clears both XSD-1.1-detection caches, not just the schemaLocation-hint one. + */ + public static void clearXsd11SchemaByNamespaceCache() { + Xsd11ValidationHelper.clearSchemaCache(); + } + /** * Constructs a Collection Object (not yet persisted) * @@ -1125,9 +1134,8 @@ public void storeDocument(final Txn transaction, final DBBroker broker, final Xm // Store XML Document final BiConsumer2E validatorFn = (xmlReader1, validateIndexInfo) -> { - validateIndexInfo.setReader(xmlReader1, null); try { - xmlReader1.parse(source); + Xsd11ValidationHelper.parseOrValidateXmlSource(broker, xmlReader1, validateIndexInfo, source); } catch(final SAXException e) { throw new SAXException("The XML parser reported a problem: " + e.getMessage(), e); } catch(final IOException e) { @@ -1137,8 +1145,7 @@ public void storeDocument(final Txn transaction, final DBBroker broker, final Xm final BiConsumer2E parserFn = (xmlReader1, storeIndexInfo) -> { try { - storeIndexInfo.setReader(xmlReader1, null); - xmlReader1.parse(source); + Xsd11ValidationHelper.parseOrValidateXmlSource(broker, xmlReader1, storeIndexInfo, source); } catch(final IOException e) { throw new EXistException(e); } diff --git a/exist-core/src/main/java/org/exist/collections/Xsd11SchemaCache.java b/exist-core/src/main/java/org/exist/collections/Xsd11SchemaCache.java new file mode 100644 index 00000000000..cc1eb4dd425 --- /dev/null +++ b/exist-core/src/main/java/org/exist/collections/Xsd11SchemaCache.java @@ -0,0 +1,66 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.collections; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import javax.annotation.Nullable; +import javax.xml.validation.Schema; +import java.util.Optional; + +/** + * Per-namespace cache of whether the system catalog's grammar for that namespace needs an + * XSD 1.1-capable loader -- see {@link Xsd11ValidationHelper#resolveXsd11SchemaForNamespace}. + * A cache miss ({@link #get(String)} returning {@code null}) means "not yet resolved"; a hit of + * {@code Optional.empty()} means "resolved: the standard XSD 1.0 pipeline handles this namespace + * fine". + *

+ * Namespaces reaching this cache are catalog-registered (a finite, admin-controlled set, never + * attacker-influenced), so unlike {@link org.exist.validation.Xsd11SchemaDetection}'s + * location-driven cache, no eviction/bounding is needed here -- entries live for the life of the + * JVM, cleared only via {@link #clear()}. + */ +final class Xsd11SchemaCache { + + private static final Cache> CACHE = Caffeine.newBuilder().build(); + + private Xsd11SchemaCache() { + } + + /** + * @return the cached resolution for {@code namespace}, or {@code null} if nothing has been + * cached for it yet. + */ + @Nullable + static Optional get(final String namespace) { + return CACHE.getIfPresent(namespace); + } + + static void put(final String namespace, final Optional schema) { + CACHE.put(namespace, schema); + } + + static void clear() { + CACHE.invalidateAll(); + } +} diff --git a/exist-core/src/main/java/org/exist/collections/Xsd11ValidationHelper.java b/exist-core/src/main/java/org/exist/collections/Xsd11ValidationHelper.java new file mode 100644 index 00000000000..c72783d3a10 --- /dev/null +++ b/exist-core/src/main/java/org/exist/collections/Xsd11ValidationHelper.java @@ -0,0 +1,386 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.collections; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.exist.Namespaces; +import org.exist.storage.DBBroker; +import org.exist.util.SaxonConfiguration; +import org.exist.util.XMLReaderObjectFactory; +import org.exist.validation.Xsd11SchemaDetection; +import org.xml.sax.Attributes; +import org.xml.sax.ContentHandler; +import org.xml.sax.InputSource; +import org.xml.sax.Locator; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; +import org.xml.sax.ext.LexicalHandler; +import org.xmlresolver.Resolver; + +import javax.annotation.Nullable; +import javax.xml.XMLConstants; +import javax.xml.transform.Source; +import javax.xml.transform.TransformerException; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; +import javax.xml.validation.ValidatorHandler; +import java.io.IOException; +import java.util.Optional; + +/** + * Store-time XSD 1.1 validation infrastructure for {@link MutableCollection}: resolving/caching + * the XSD 1.1 {@link Schema} a document needs (if any), and driving validation against it. + *

+ * Package-private and static-only -- this is validation/schema infrastructure, not Collection + * state, split out of {@link MutableCollection} per review discussion on + * #6530/#6551. + * + * @see #5541 + */ +final class Xsd11ValidationHelper { + + private static final Logger LOG = LogManager.getLogger(Xsd11ValidationHelper.class); + + private Xsd11ValidationHelper() { + } + + /** + * Discards all cached {@link #resolveXsd11SchemaForNamespace} results -- called alongside + * {@code Jaxp.clearXsd11DetectionCache()} by {@code validation:clear-grammar-cache()} (via + * {@link MutableCollection#clearXsd11SchemaByNamespaceCache()}) so one admin action clears both + * XSD-1.1-detection caches, not just the schemaLocation-hint one. + */ + static void clearSchemaCache() { + Xsd11SchemaCache.clear(); + } + + /** + * Lazily compiles, once per JVM, the no-pre-supplied-source XSD 1.1 {@link Schema} used for the + * case where the instance carries its own {@code xsi:schemaLocation}/{@code + * noNamespaceSchemaLocation} hint that needs XSD 1.1 to load (detected via + * {@link Xsd11SchemaDetection#detectXsd11ViaSchemaLocation}): unlike + * {@link #resolveXsd11SchemaForNamespace(Resolver, String)}, no pre-supplied Source is needed + * here, since dynamic discovery can follow the instance's own hint itself, the same way the + * default SAX pipeline follows it for XSD 1.0. + *

+ * Initialization-on-demand holder idiom: thread-safe with no explicit synchronization, relying + * on the JVM's class-initialization guarantees instead of manual double-checked locking. + */ + private static final class Xsd11DynamicDiscoverySchemaHolder { + private static final Schema INSTANCE; + static { + try { + INSTANCE = SchemaFactory.newInstance(Namespaces.XSD_1_1_NS).newSchema(); + } catch (final SAXException e) { + throw new ExceptionInInitializerError(e); + } + } + } + + private static Schema getXsd11DynamicDiscoverySchema() { + return Xsd11DynamicDiscoverySchemaHolder.INSTANCE; + } + + /** + * Resolves the system catalog's grammar for {@code namespace} and, only if that grammar + * actually requires XSD 1.1 to load, compiles and caches an explicit-Source XSD 1.1 + * {@link Schema} for it; returns {@code null} when the resolved grammar loads fine under the + * standard XSD 1.0 {@link SchemaFactory} (the overwhelmingly common case), so the default SAX + * pipeline keeps handling that case exactly as before. + * + *

"Does it need 1.1?" is decided empirically -- attempt the cheap, side-effect-free compile + * via the plain (1.0) {@code SchemaFactory} first; if that throws, the grammar needs an + * XSD-1.1-aware loader. This is the only reliable signal here: the W3C XSD 1.1 meta-schema + * itself (the motivating case, see {@code https://github.com/eXist-db/exist/issues/5541}) does + * not self-declare {@code vc:minVersion} the way a hand-authored 1.1 schema would, so peeking + * for that attribute (as {@link Xsd11SchemaDetection} does for the + * schemaLocation-hint case below) does not apply to namespace-only resolution.

+ * + *

A no-pre-supplied-source {@code Schema}'s dynamic discovery only resolves grammars via an + * instance's own {@code xsi:schemaLocation} hint, not via "this document's root namespace has + * no hint at all, but happens to be a namespace the catalog can resolve" (confirmed + * empirically -- a no-source {@code Schema} fails with {@code cvc-elt.1.a} for exactly this + * case, even with the resolver wired on) -- hence the explicit {@code Source} here.

+ */ + @Nullable + private static Schema resolveXsd11SchemaForNamespace(final Resolver catalogResolver, final String namespace) throws SAXException { + final Optional cached = Xsd11SchemaCache.get(namespace); + if (cached != null) { + return cached.orElse(null); + } + + Schema result = null; + try { + final Source probeSource = catalogResolver.resolve(namespace, null); + if (probeSource != null) { + try { + SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI).newSchema(probeSource); + // Loads fine under the standard (1.0) pipeline -- nothing special needed. + } catch (final SAXException loadsAs10Failure) { + final Source compileSource = catalogResolver.resolve(namespace, null); + if (compileSource != null) { + final SchemaFactory xsd11Factory = SchemaFactory.newInstance(Namespaces.XSD_1_1_NS); + xsd11Factory.setResourceResolver(catalogResolver); + result = xsd11Factory.newSchema(compileSource); + } + } + } + } catch (final TransformerException e) { + throw new SAXException(e); + } + + Xsd11SchemaCache.put(namespace, Optional.ofNullable(result)); + return result; + } + + /** + * Parses/validates {@code source} via {@code xmlReader1} as before, unless the document needs + * an XSD 1.1-capable loader -- the bundled Xerces fork's XSD 1.1 support is only wired into + * the JAXP {@code SchemaFactory}/{@code Validator} API, never into this dynamic-discovery SAX + * pipeline (confirmed empirically: setting Xerces' internal {@code schema/version} property on + * a standard {@code XMLReader} throws {@code SAXNotRecognizedException}). Two independent ways + * this can be needed, checked up front from a single peek of the root element via + * {@link Xsd11SchemaDetection#peekRootElement(InputSource)} (never via + * retry-after-failure: by the time a SAX parse fails, the {@link IndexInfo}'s + * {@link org.exist.Indexer}/triggers have already received partial events for an aborted + * document, so re-feeding them via a second pass is not safe): + * + *
    + *
  1. The document being stored is itself a schema document (root element in the W3C XML + * Schema namespace, no {@code schemaLocation} hint at all): see + * {@link #resolveXsd11SchemaForNamespace(Resolver, String)}.
    2. + *
  2. The instance carries its own {@code xsi:schemaLocation}/{@code + * noNamespaceSchemaLocation} hint that resolves to a schema declaring {@code + * vc:minVersion="1.1"}: see {@link Xsd11SchemaDetection}, shared with + * {@code validation:jaxp()}'s own up-front peek.
    3. + *
+ * + *

Anything neither case catches (most prominently: a catalog-mediated {@code + * schemaLocation} hint whose target doesn't self-declare {@code vc:minVersion}) falls through + * to the default pipeline unchanged, and may still fail with {@code cvc-elt.1.a} -- a known, + * accepted limitation of peek-only detection with no retry safety net.

+ * + *

The peek above, and the validate/store double-invocation of this method itself (once via + * {@code validatorFn}, once via {@code parserFn} in {@code MutableCollection.storeXmlDocument}), + * both read {@code source} more than once via the same {@link InputSource#getByteStream()}/ + * {@link InputSource#getCharacterStream()} methods. Safe because every {@link InputSource} + * actually reaching {@code storeDocument()}/{@code storeXmlDocument()} today (see {@code + * org.exist.util.StringInputSource} and its siblings) vends a fresh stream per call by + * convention -- a precondition of this method, not something it (or its callers) re-validates.

+ * + * @see #5541 + */ + static void parseOrValidateXmlSource(final DBBroker broker, final XMLReader xmlReader1, final IndexInfo indexInfo, final InputSource source) throws SAXException, IOException { + indexInfo.setReader(xmlReader1, null); + + boolean schemaValidationEnabled; + try { + schemaValidationEnabled = xmlReader1.getFeature(XMLReaderObjectFactory.APACHE_FEATURES_VALIDATION_SCHEMA); + } catch (final SAXException e) { + schemaValidationEnabled = false; + LOG.debug("Could not determine if schema validation is enabled, assuming disabled: {}", e.getMessage()); + } + + if (schemaValidationEnabled) { + final Resolver catalogResolver = SaxonConfiguration.resolveCatalogResolver(broker.getBrokerPool().getConfiguration()); + + // One StAX pass yields both the root namespace (case 1) and, if case 1 doesn't apply, + // the root attributes detectXsd11ViaSchemaLocation needs (case 2) -- avoids peeking the + // same root start tag twice. + final Xsd11SchemaDetection.RootElementInfo rootElement = Xsd11SchemaDetection.peekRootElement(source); + final String rootNamespace = rootElement == null ? null : rootElement.namespaceUri(); + if (catalogResolver != null && rootNamespace != null) { + final Schema schema = resolveXsd11SchemaForNamespace(catalogResolver, rootNamespace); + if (schema != null) { + validateWithXsd11Schema(xmlReader1, schema, catalogResolver, indexInfo, source); + return; + } + } + + if (Xsd11SchemaDetection.detectXsd11ViaSchemaLocation(broker.getCurrentSubject().getName(), rootElement, source.getSystemId())) { + validateWithXsd11Schema(xmlReader1, getXsd11DynamicDiscoverySchema(), catalogResolver, indexInfo, source); + return; + } + } + + xmlReader1.parse(source); + } + + /** + * Validates {@code source} against {@code schema}, feeding the resulting SAX events into + * {@code indexInfo}'s indexing/trigger pipeline. + *

+ * Uses {@link Schema#newValidatorHandler()} driven by {@code xmlReader1.parse(source)} rather + * than {@link Schema#newValidator()}'s {@code validate(Source, Result)} -- a {@link + * javax.xml.transform.sax.SAXResult SAXResult} has no lexical-handler hook (confirmed by + * inspecting the bundled Xerces fork's {@code ValidatorImpl}: it wires a {@code SAXResult}'s + * {@code ContentHandler} but never a {@code LexicalHandler}), so comments/CDATA sections would + * otherwise be silently dropped on this path, unlike the default {@code xmlReader1.parse(source)} + * path which {@link IndexInfo#setReader} already wires with both. + *

+ * {@link ValidatorHandler} itself has no public API to register a downstream + * {@link LexicalHandler} (confirmed empirically: it does not forward comment()/startCDATA()/ + * endCDATA() to its registered {@code ContentHandler} even when that handler also implements + * {@code LexicalHandler}), so {@link Xsd11LexicalHandlerForwarder} sits in front of it, + * forwarding content events to the validator (so validation/indexing both still happen) and + * lexical events directly to {@code indexInfo}'s real lexical handler, bypassing the validator + * for those (comments/CDATA boundaries are not part of any XSD content model -- CDATA's + * character content is still validated normally, via the ordinary {@code characters()} event). + *

+ * Reuses the caller's already-configured {@code xmlReader1} (rather than constructing a fresh + * {@link XMLReader}) purely to drive the parse; its content/lexical handlers are repointed at + * {@link Xsd11LexicalHandlerForwarder} for the duration of this call. + */ + private static void validateWithXsd11Schema(final XMLReader xmlReader1, final Schema schema, @Nullable final Resolver catalogResolver, final IndexInfo indexInfo, final InputSource source) throws SAXException, IOException { + final ValidatorHandler validatorHandler = schema.newValidatorHandler(); + if (catalogResolver != null) { + validatorHandler.setResourceResolver(catalogResolver); + } + validatorHandler.setErrorHandler(indexInfo.getIndexer()); + validatorHandler.setContentHandler(indexInfo.getContentHandler()); + final Xsd11LexicalHandlerForwarder forwarder = new Xsd11LexicalHandlerForwarder(validatorHandler, indexInfo.getLexicalHandler()); + + // xmlReader1 is the pooled, dynamic-discovery-validating reader -- its own schema + // validation feature must be off for this call, or it independently (mis)validates the + // document against the default XSD 1.0 pipeline in parallel with validatorHandler's XSD + // 1.1 validation, producing spurious cvc-elt.1.a/s4s-att-not-allowed errors. Saved and + // restored rather than left disabled, since storeXmlDocument() reuses this same xmlReader1 + // instance for a second, separate parseOrValidateXmlSource() call (the store phase). + final boolean wasValidating = xmlReader1.getFeature(XMLReaderObjectFactory.APACHE_FEATURES_VALIDATION_SCHEMA); + try { + xmlReader1.setFeature(XMLReaderObjectFactory.APACHE_FEATURES_VALIDATION_SCHEMA, false); + xmlReader1.setFeature(Namespaces.SAX_VALIDATION, false); + xmlReader1.setFeature(Namespaces.SAX_VALIDATION_DYNAMIC, false); + + xmlReader1.setContentHandler(forwarder); + xmlReader1.setProperty(Namespaces.SAX_LEXICAL_HANDLER, forwarder); + xmlReader1.parse(source); + } finally { + xmlReader1.setFeature(XMLReaderObjectFactory.APACHE_FEATURES_VALIDATION_SCHEMA, wasValidating); + xmlReader1.setFeature(Namespaces.SAX_VALIDATION, wasValidating); + xmlReader1.setFeature(Namespaces.SAX_VALIDATION_DYNAMIC, wasValidating); + } + } + + /** + * Splits incoming SAX events from a single source (an {@link XMLReader}) across two + * downstream consumers: content events go to {@code contentDelegate} (a {@link + * ValidatorHandler}, so schema validation still happens), lexical events go directly to + * {@code lexicalDelegate}, bypassing the validator -- see {@link + * #validateWithXsd11Schema(XMLReader, Schema, Resolver, IndexInfo, InputSource)} for why. + */ + private record Xsd11LexicalHandlerForwarder(ContentHandler contentDelegate, LexicalHandler lexicalDelegate) + implements ContentHandler, LexicalHandler { + + @Override + public void setDocumentLocator(final Locator locator) { + contentDelegate.setDocumentLocator(locator); + } + + @Override + public void startDocument() throws SAXException { + contentDelegate.startDocument(); + } + + @Override + public void endDocument() throws SAXException { + contentDelegate.endDocument(); + } + + @Override + public void startPrefixMapping(final String prefix, final String uri) throws SAXException { + contentDelegate.startPrefixMapping(prefix, uri); + } + + @Override + public void endPrefixMapping(final String prefix) throws SAXException { + contentDelegate.endPrefixMapping(prefix); + } + + @Override + public void startElement(final String uri, final String localName, final String qName, final Attributes atts) throws SAXException { + contentDelegate.startElement(uri, localName, qName, atts); + } + + @Override + public void endElement(final String uri, final String localName, final String qName) throws SAXException { + contentDelegate.endElement(uri, localName, qName); + } + + @Override + public void characters(final char[] ch, final int start, final int length) throws SAXException { + contentDelegate.characters(ch, start, length); + } + + @Override + public void ignorableWhitespace(final char[] ch, final int start, final int length) throws SAXException { + contentDelegate.ignorableWhitespace(ch, start, length); + } + + @Override + public void processingInstruction(final String target, final String data) throws SAXException { + contentDelegate.processingInstruction(target, data); + } + + @Override + public void skippedEntity(final String name) throws SAXException { + contentDelegate.skippedEntity(name); + } + + @Override + public void startDTD(final String name, final String publicId, final String systemId) throws SAXException { + lexicalDelegate.startDTD(name, publicId, systemId); + } + + @Override + public void endDTD() throws SAXException { + lexicalDelegate.endDTD(); + } + + @Override + public void startEntity(final String name) throws SAXException { + lexicalDelegate.startEntity(name); + } + + @Override + public void endEntity(final String name) throws SAXException { + lexicalDelegate.endEntity(name); + } + + @Override + public void startCDATA() throws SAXException { + lexicalDelegate.startCDATA(); + } + + @Override + public void endCDATA() throws SAXException { + lexicalDelegate.endCDATA(); + } + + @Override + public void comment(final char[] ch, final int start, final int length) throws SAXException { + lexicalDelegate.comment(ch, start, length); + } + } +} diff --git a/exist-core/src/main/java/org/exist/http/Descriptor.java b/exist-core/src/main/java/org/exist/http/Descriptor.java index 571aa558276..1f284d76b15 100644 --- a/exist-core/src/main/java/org/exist/http/Descriptor.java +++ b/exist-core/src/main/java/org/exist/http/Descriptor.java @@ -28,6 +28,7 @@ import org.exist.dom.memtree.SAXAdapter; import org.exist.util.ConfigurationHelper; import org.exist.util.ExistSAXParserFactory; +import org.exist.util.SchemaVersion; import org.exist.util.SingleInstanceConfiguration; import org.exist.xquery.Expression; import org.w3c.dom.Document; @@ -138,6 +139,8 @@ private Descriptor() { final Document doc = adapter.getDocument(); + SchemaVersion.logDocumentVersion(LOG, doc.getDocumentElement(), SchemaVersion.DESCRIPTOR, "descriptor.xml"); + //load attribue settings if ("true".equals(doc.getDocumentElement().getAttribute("request-replay-log"))) { final Path logFile = Path.of("request-replay-log.txt"); diff --git a/exist-core/src/main/java/org/exist/http/urlrewrite/RewriteConfig.java b/exist-core/src/main/java/org/exist/http/urlrewrite/RewriteConfig.java index 745dd98d106..a08a016c20b 100644 --- a/exist-core/src/main/java/org/exist/http/urlrewrite/RewriteConfig.java +++ b/exist-core/src/main/java/org/exist/http/urlrewrite/RewriteConfig.java @@ -35,6 +35,7 @@ import net.sf.saxon.str.StringView; import net.sf.saxon.trans.XPathException; import org.exist.util.XMLReaderPool; +import org.exist.util.SchemaVersion; import org.exist.xmldb.XmldbURI; import org.exist.xquery.Constants; import org.exist.xquery.Expression; @@ -199,6 +200,7 @@ private void configure(final String controllerConfig) throws ServletException { private void parse(final Document doc) throws ServletException { final Element root = doc.getDocumentElement(); + SchemaVersion.logDocumentVersion(LOG, root, SchemaVersion.CONTROLLER_CONFIG, "controller-config.xml"); Node child = root.getFirstChild(); while (child != null) { final String ns = child.getNamespaceURI(); diff --git a/exist-core/src/main/java/org/exist/resolver/ResolverFactory.java b/exist-core/src/main/java/org/exist/resolver/ResolverFactory.java index 9fa16cd3414..c55869424c0 100644 --- a/exist-core/src/main/java/org/exist/resolver/ResolverFactory.java +++ b/exist-core/src/main/java/org/exist/resolver/ResolverFactory.java @@ -84,11 +84,7 @@ public interface ResolverFactory { * @throws URISyntaxException if one of the catalog URI is invalid */ static Resolver newResolver(final List>> catalogs) throws URISyntaxException { - final XMLResolverConfiguration resolverConfiguration = new XMLResolverConfiguration(); - resolverConfiguration.setFeature(ResolverFeature.RESOLVER_LOGGER_CLASS, "org.xmlresolver.logging.SystemLogger"); - resolverConfiguration.setFeature(ResolverFeature.CATALOG_LOADER_CLASS, "org.xmlresolver.loaders.ValidatingXmlLoader"); - resolverConfiguration.setFeature(ResolverFeature.CLASSPATH_CATALOGS, true); - resolverConfiguration.setFeature(ResolverFeature.URI_FOR_SYSTEM, true); + final XMLResolverConfiguration resolverConfiguration = newCatalogConfiguration(); for (final Tuple2> catalog : catalogs) { String strCatalogUri = catalog._1; @@ -119,11 +115,7 @@ static Resolver newResolver(final List>> ca * @throws URISyntaxException if one of the catalog URI is invalid */ static Resolver newResolverFromSax(final List>> catalogs) throws URISyntaxException { - final XMLResolverConfiguration resolverConfiguration = new XMLResolverConfiguration(); - resolverConfiguration.setFeature(ResolverFeature.RESOLVER_LOGGER_CLASS, "org.xmlresolver.logging.SystemLogger"); - resolverConfiguration.setFeature(ResolverFeature.CATALOG_LOADER_CLASS, "org.xmlresolver.loaders.ValidatingXmlLoader"); - resolverConfiguration.setFeature(ResolverFeature.CLASSPATH_CATALOGS, true); - resolverConfiguration.setFeature(ResolverFeature.URI_FOR_SYSTEM, true); + final XMLResolverConfiguration resolverConfiguration = newCatalogConfiguration(); final CatalogManager manager = resolverConfiguration.getFeature(ResolverFeature.CATALOG_MANAGER); @@ -148,6 +140,24 @@ static Resolver newResolverFromSax(final List existHomeDi final Document doc = adapter.getDocument(); + SchemaVersion.logDocumentVersion(LOG, doc.getDocumentElement(), SchemaVersion.CONF, + configFilePath.map(p -> "conf.xml (" + p + ")").orElse("conf.xml")); + //indexer settings configureElement(doc, Indexer.CONFIGURATION_ELEMENT_NAME, element -> configureIndexer(doc, element)); //scheduler settings diff --git a/exist-core/src/main/java/org/exist/util/MimeTable.java b/exist-core/src/main/java/org/exist/util/MimeTable.java index 09d303e9ae9..87252ceee43 100644 --- a/exist-core/src/main/java/org/exist/util/MimeTable.java +++ b/exist-core/src/main/java/org/exist/util/MimeTable.java @@ -143,8 +143,9 @@ public MimeTable(final Path path) { } try (final InputStream is = Files.newInputStream(path)) { LOG.info("Loading mime table from file: {}", path.toAbsolutePath()); - loadMimeTypes(is); - this.src = path.toUri().toString(); + final String sourceDescription = path.toUri().toString(); + loadMimeTypes(is, sourceDescription); + this.src = sourceDescription; } catch (final ParserConfigurationException | SAXException | IOException e) { throw new IllegalStateException(FILE_LOAD_FAILED_ERR + path.toAbsolutePath(), e); } @@ -264,7 +265,7 @@ private void load(final InputStream stream, final String src) { private void loadFromStream(final InputStream stream, final String sourceDescription) { try (stream) { - loadMimeTypes(stream); + loadMimeTypes(stream, sourceDescription); this.src = sourceDescription; } catch (final ParserConfigurationException | SAXException | IOException e) { throw new IllegalStateException("Failed to load mime-type table from " + sourceDescription, e); @@ -275,16 +276,17 @@ private void loadFromStream(final InputStream stream, final String sourceDescrip * Load Mime Types * * @param stream input stream. + * @param sourceDescription description of the stream's origin, for diagnostic messages. * * @throws SAXException if an error occurs whilst reading the XML stream * @throws ParserConfigurationException if an error occurs whilst parsing the stream * @throws IOException if an error occurs whilst reading the stream */ - private void loadMimeTypes(final InputStream stream) throws ParserConfigurationException, SAXException, IOException { + private void loadMimeTypes(final InputStream stream, final String sourceDescription) throws ParserConfigurationException, SAXException, IOException { final SAXParserFactory factory = ExistSAXParserFactory.getSAXParserFactory(); factory.setNamespaceAware(true); factory.setValidating(false); - final InputSource src = new InputSource(stream); + final InputSource inputSource = new InputSource(stream); final SAXParser parser = factory.newSAXParser(); final XMLReader reader = parser.getXMLReader(); @@ -292,8 +294,8 @@ private void loadMimeTypes(final InputStream stream) throws ParserConfigurationE reader.setFeature("http://xml.org/sax/features/external-parameter-entities", false); reader.setFeature(FEATURE_SECURE_PROCESSING, true); - reader.setContentHandler(new MimeTableHandler()); - reader.parse(src); + reader.setContentHandler(new MimeTableHandler(sourceDescription)); + reader.parse(inputSource); } private class MimeTableHandler extends DefaultHandler { @@ -302,16 +304,23 @@ private class MimeTableHandler extends DefaultHandler { private static final String DESCRIPTION = "description"; private static final String MIME_TYPE = "mime-type"; private static final String MIME_TYPES = "mime-types"; - + + private final String sourceDescription; private MimeType mime = null; private final StringBuilder charBuf = new StringBuilder(64); + MimeTableHandler(final String sourceDescription) { + this.sourceDescription = sourceDescription; + } + @Override public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { if (MIME_TYPES.equals(qName)) { + SchemaVersion.logDocumentVersion(LOG, attributes.getValue(SchemaVersion.ATTRIBUTE), + SchemaVersion.MIME_TYPES, sourceDescription != null ? "mime-types.xml (" + sourceDescription + ")" : "mime-types.xml"); // Check for a default mime type settings final String defaultMimeAttr = attributes.getValue("default-mime-type"); final String defaultTypeAttr = attributes.getValue("default-resource-type"); diff --git a/exist-core/src/main/java/org/exist/util/SaxonConfiguration.java b/exist-core/src/main/java/org/exist/util/SaxonConfiguration.java index 16fc989cd8b..e61499b8c66 100644 --- a/exist-core/src/main/java/org/exist/util/SaxonConfiguration.java +++ b/exist-core/src/main/java/org/exist/util/SaxonConfiguration.java @@ -22,12 +22,18 @@ package org.exist.util; import net.jcip.annotations.ThreadSafe; +import net.sf.saxon.lib.ResourceRequest; +import net.sf.saxon.lib.ResourceResolver; import net.sf.saxon.s9api.Processor; import net.sf.saxon.trans.XPathException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.exist.storage.BrokerPool; +import org.xmlresolver.Resolver; +import javax.annotation.Nullable; +import javax.xml.transform.Source; +import javax.xml.transform.TransformerException; import javax.xml.transform.stream.StreamSource; import java.io.IOException; import java.nio.file.Files; @@ -50,11 +56,48 @@ public final class SaxonConfiguration { private final net.sf.saxon.Configuration configuration; private final Processor processor; - private SaxonConfiguration(final net.sf.saxon.Configuration configuration) { + private SaxonConfiguration(final net.sf.saxon.Configuration configuration, final BrokerPool brokerPool) { this.configuration = configuration; this.processor = new Processor(configuration); - //TODO (AP) This is a better place to configure URI/Resource resolution for Saxon within eXist - //At present the configuration for Saxon to resolve xmldb:exist: URIs is restricted to fn:transform + + // System catalog (webapp/WEB-INF/catalog.xml by default) as the Saxon-wide fallback + // resource resolver -- governs doc()/document() inside XSLT, and anything xmldb:exist:-aware + // resolvers (wired separately per-call in fn:transform/transform:transform) don't claim (#350). + final Resolver catalogResolver = resolveCatalogResolver(brokerPool.getConfiguration()); + if (catalogResolver != null) { + configuration.setResourceResolver(new CatalogResourceResolver(catalogResolver)); + } + } + + /** + * Fetches the system catalog {@link Resolver} from {@code configuration}, if one is configured. + * Shared accessor for the {@code (Resolver) configuration.getProperty(XMLReaderObjectFactory.CATALOG_RESOLVER)} + * cast otherwise repeated independently at each call site -- a key/type change to that property + * only needs updating here. + * + * @return the configured catalog resolver, or {@code null} if none is configured. + */ + @Nullable + public static Resolver resolveCatalogResolver(final Configuration configuration) { + final Object resolver = configuration.getProperty(XMLReaderObjectFactory.CATALOG_RESOLVER); + return resolver instanceof Resolver r ? r : null; + } + + /** + * Adapts a classic {@link javax.xml.transform.URIResolver} (here, the system catalog) to + * Saxon 10+'s {@link ResourceResolver}, which {@link net.sf.saxon.Configuration} requires -- + * it no longer exposes a plain {@code setURIResolver(URIResolver)}. + */ + private record CatalogResourceResolver(Resolver delegate) implements ResourceResolver { + + @Override + public Source resolve(final ResourceRequest request) throws XPathException { + try { + return delegate.resolve(request.relativeUri, request.baseUri); + } catch (final TransformerException e) { + throw new XPathException(e); + } + } } /** @@ -104,7 +147,7 @@ public static SaxonConfiguration loadConfiguration(final BrokerPool brokerPool) saxonConfiguration.ifPresent(SaxonConfiguration::reportLicensedFeatures); - return new SaxonConfiguration(saxonConfiguration.get()); + return new SaxonConfiguration(saxonConfiguration.get(), brokerPool); } static private Optional readSaxonConfigurationFile(final Path saxonConfigFile) { diff --git a/exist-core/src/main/java/org/exist/util/SchemaVersion.java b/exist-core/src/main/java/org/exist/util/SchemaVersion.java new file mode 100644 index 00000000000..bd472dc384d --- /dev/null +++ b/exist-core/src/main/java/org/exist/util/SchemaVersion.java @@ -0,0 +1,73 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.util; + +import org.apache.logging.log4j.Logger; +import org.w3c.dom.Element; + +/** + * Optional {@code schemaVersion} on native config instance documents — mirrors + * {@code xs:schema/@version} on the paired XSD (native schema semver, not eXist product version). + */ +public final class SchemaVersion { + + public static final String ATTRIBUTE = "schemaVersion"; + + /** + * Paired {@code xs:schema/@version} values for canonical templates -- generated at build time + * from {@code schema/*.xsd} itself (see {@link GeneratedSchemaVersions}), so these can never + * drift from the schemas they describe. + */ + public static final String CONF = GeneratedSchemaVersions.CONF; + public static final String COLLECTION_XCONF = GeneratedSchemaVersions.COLLECTION_XCONF; + public static final String DESCRIPTOR = GeneratedSchemaVersions.DESCRIPTOR; + public static final String MIME_TYPES = GeneratedSchemaVersions.MIME_TYPES; + public static final String CONTROLLER_CONFIG = GeneratedSchemaVersions.CONTROLLER_CONFIG; + + private SchemaVersion() { + } + + /** + * Log when {@code schemaVersion} is missing (legacy) or differs from the schema version this build expects. + */ + public static void logDocumentVersion(final Logger log, final Element root, + final String expectedVersion, final String documentDescription) { + logDocumentVersion(log, root != null ? root.getAttribute(ATTRIBUTE) : "", expectedVersion, documentDescription); + } + + /** + * SAX variant when only the attribute value is available. + */ + public static void logDocumentVersion(final Logger log, final String declaredVersion, + final String expectedVersion, final String documentDescription) { + if (declaredVersion == null || declaredVersion.isEmpty()) { + log.debug("{} has no {} attribute (legacy document)", documentDescription, ATTRIBUTE); + return; + } + if (!declaredVersion.equals(expectedVersion)) { + log.warn("{} declares {}=\"{}\" but this eXist build expects \"{}\"", + documentDescription, ATTRIBUTE, declaredVersion, expectedVersion); + } else { + log.debug("{} {}=\"{}\"", documentDescription, ATTRIBUTE, declaredVersion); + } + } +} diff --git a/exist-core/src/main/java/org/exist/validation/Validator.java b/exist-core/src/main/java/org/exist/validation/Validator.java index 4766bfbc8d0..7ba8cc05a81 100644 --- a/exist-core/src/main/java/org/exist/validation/Validator.java +++ b/exist-core/src/main/java/org/exist/validation/Validator.java @@ -46,6 +46,10 @@ import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; +import javax.xml.validation.ValidatorHandler; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URISyntaxException; @@ -64,6 +68,16 @@ public class Validator { private static final Logger logger = LogManager.getLogger(Validator.class); + private static final String XSD_1_1_NS = "http://www.w3.org/XML/XMLSchema/v1.1"; + + /** + * Generous upper bound on how much of the document's prolog (up to and including the root + * start tag) {@link #validateParse(InputStream, String, String)}'s XSD 1.1 peek may read + * before giving up on {@code mark()}/{@code reset()} -- real documents' prologs are at most a + * few KB even with many namespace declarations/attributes. + */ + private static final int XSD11_PEEK_MARK_LIMIT = 65536; + private final BrokerPool brokerPool; private final Subject subject; private final GrammarPool grammarPool; @@ -115,6 +129,23 @@ public ValidationReport validate(final InputStream stream) { * @return Validation report containing all validation info. */ public ValidationReport validate(final InputStream stream, @Nullable String grammarUrl) { + return validate(stream, grammarUrl, null); + } + + /** + * Validate XML data from reader using specified grammar. + * + * @param grammarUrl User supplied path to grammar, or null. + * @param stream XML input. + * @param documentBaseUri the base URI of {@code stream}'s document (e.g. the stored document's + * own URI), or {@code null} if unknown -- used only to resolve the + * instance's own {@code xsi:schemaLocation} hint (see {@link + * Xsd11SchemaDetection#detectXsd11ViaSchemaLocation}) when deciding + * whether an XSD 1.1-capable validator is needed; validation against an + * explicitly-supplied {@code grammarUrl} does not otherwise depend on it. + * @return Validation report containing all validation info. + */ + public ValidationReport validate(final InputStream stream, @Nullable String grammarUrl, @Nullable final String documentBaseUri) { // repair path to local resource if (grammarUrl != null) { @@ -129,7 +160,7 @@ public ValidationReport validate(final InputStream stream, @Nullable String gram } else { // Validate with Xerces - return validateParse(stream, grammarUrl); + return validateParse(stream, grammarUrl, documentBaseUri); } } @@ -195,14 +226,41 @@ public ValidationReport validateParse(final InputStream stream) { * @param stream XML input. * @return Validation report containing all validation info. */ - public ValidationReport validateParse(final InputStream stream, String grammarUrl) { + public ValidationReport validateParse(final InputStream stream, final String grammarUrl) { + return validateParse(stream, grammarUrl, null); + } + + /** + * Validate XML data from reader using specified grammar. + * + * @param grammarUrl User supplied path to grammar. + * @param stream XML input. + * @param documentBaseUri see {@link #validate(InputStream, String, String)}. + * @return Validation report containing all validation info. + */ + public ValidationReport validateParse(final InputStream stream, final String grammarUrl, @Nullable final String documentBaseUri) { logger.debug("Start validation."); final ValidationReport report = new ValidationReport(); final ValidationContentHandler contenthandler = new ValidationContentHandler(); + final BufferedInputStream bufferedStream = new BufferedInputStream(stream, XSD11_PEEK_MARK_LIMIT); + final ValidationReport xsd11Report = tryValidateWithXsd11Schema(bufferedStream, documentBaseUri, contenthandler, report); + if (xsd11Report != null) { + return xsd11Report; + } + + return validateParseDefault(bufferedStream, grammarUrl, contenthandler, report); + } + /** + * The XSD 1.0/DTD dynamic-discovery SAX pipeline {@link #validateParse} falls through to when + * no XSD 1.1 hint was detected (or could be resolved) -- unchanged behavior, just extracted + * out of {@code validateParse} to keep that method's own branching to one decision (XSD 1.1 + * or not). + */ + private ValidationReport validateParseDefault(final InputStream stream, String grammarUrl, final ValidationContentHandler contenthandler, final ValidationReport report) { try { final XMLReader xmlReader = getXMLReader(contenthandler, report); @@ -276,6 +334,117 @@ public ValidationReport validateParse(final InputStream stream, String grammarUr return report; } + /** + * The bundled Xerces fork's XSD 1.1 support is only wired into the JAXP + * SchemaFactory/Validator API, never into the dynamic-discovery SAX pipeline {@code + * validateParse}'s default path uses -- same limitation {@code org.exist.collections. + * MutableCollection}'s store-time validation and {@code validation:jaxp()} already work + * around. Peeks the instance's own {@code xsi:schemaLocation}/{@code + * noNamespaceSchemaLocation} hint up front (mark/reset on {@code bufferedStream} since + * callers hand in a single-use {@link InputStream}, not a re-readable {@link InputSource}) + * and, if it resolves to a schema declaring {@code vc:minVersion="1.1"}, validates with an + * XSD 1.1-capable {@link ValidatorHandler} instead. + * + * @return the completed {@link ValidationReport} if the XSD 1.1 path was taken, or {@code + * null} if {@code documentBaseUri} is unknown (needed to resolve the hint) or no hint + * was found -- the caller should fall through to the default pipeline in that case + * (same accepted, documented limitation as the other two call sites). + */ + @Nullable + private ValidationReport tryValidateWithXsd11Schema(final BufferedInputStream bufferedStream, @Nullable final String documentBaseUri, + final ValidationContentHandler contenthandler, final ValidationReport report) { + if (documentBaseUri == null) { + return null; + } + try { + bufferedStream.mark(XSD11_PEEK_MARK_LIMIT); + final InputSource peekSource = new InputSource(bufferedStream); + final Xsd11SchemaDetection.RootElementInfo rootElement = Xsd11SchemaDetection.peekRootElement(peekSource); + bufferedStream.reset(); + + if (Xsd11SchemaDetection.detectXsd11ViaSchemaLocation(subject.getName(), rootElement, documentBaseUri)) { + logger.debug("Detected XSD 1.1 schema (vc:minVersion) via the instance's schemaLocation hint; using the XSD 1.1 validator directly."); + return validateWithXsd11Schema(bufferedStream, documentBaseUri, contenthandler, report); + } + } catch (final IOException ex) { + // Mark/reset failure (e.g. the prolog before the root start tag exceeded + // XSD11_PEEK_MARK_LIMIT) -- not a validation failure, just means the peek couldn't + // run; fall through to the default pipeline exactly as if no hint had been found. + logger.debug("Could not peek root element for XSD 1.1 detection: {}", ex.getMessage()); + } + return null; + } + + /** + * Validates {@code stream} against a no-pre-supplied-source XSD 1.1 {@link Schema} (dynamic + * discovery follows the instance's own {@code xsi:schemaLocation} hint, the same way the + * default SAX pipeline follows it for XSD 1.0). Drives a plain, non-validating {@link + * XMLReader} rather than reusing {@link #getXMLReader} -- this class builds a fresh reader per + * call rather than reusing a pooled one, so there's no risk of it independently + * (mis)validating the document via its own XSD-1.0-only dynamic discovery in parallel with + * this {@link ValidatorHandler} (the failure mode {@code MutableCollection}'s equivalent fix + * had to specifically guard against, since it reuses a pooled, already-validating reader). + * {@code contenthandler} doesn't implement {@link org.xml.sax.ext.LexicalHandler}, so unlike + * {@code MutableCollection}'s fix, no lexical-event forwarder is needed here. + */ + private ValidationReport validateWithXsd11Schema(final InputStream stream, final String documentBaseUri, final ValidationContentHandler contenthandler, final ValidationReport report) { + try { + final ValidatorHandler validatorHandler = getXsd11DynamicDiscoverySchema().newValidatorHandler(); + if (systemCatalogResolver != null) { + validatorHandler.setResourceResolver(systemCatalogResolver); + } + validatorHandler.setErrorHandler(report); + validatorHandler.setContentHandler(contenthandler); + + final SAXParserFactory saxFactory = ExistSAXParserFactory.getSAXParserFactory(); + saxFactory.setNamespaceAware(true); + final XMLReader xmlReader = saxFactory.newSAXParser().getXMLReader(); + xmlReader.setFeature(FEATURE_SECURE_PROCESSING, true); + xmlReader.setContentHandler(validatorHandler); + + // Dynamic discovery resolves the instance's own xsi:schemaLocation hint against the + // InputSource's systemId during the parse -- without it, a relative hint falls back + // to the JVM's current working directory instead of documentBaseUri (confirmed + // empirically: this was the actual cause of an early version of this fix silently + // falling through to "schema document not found" instead of validating). + final InputSource source = new InputSource(stream); + source.setSystemId(documentBaseUri); + + report.start(); + xmlReader.parse(source); + report.stop(); + + report.setNamespaceUri(contenthandler.getNamespaceUri()); + } catch (final ParserConfigurationException | SAXException | IOException ex) { + logger.error(ex); + report.setThrowable(ex); + } finally { + report.stop(); + logger.debug("Validation performed in {} msec.", report.getValidationDuration()); + } + return report; + } + + /** + * Lazily compiles, once per JVM, the no-pre-supplied-source XSD 1.1 {@link Schema} used by + * {@link #validateWithXsd11Schema(InputStream, ValidationContentHandler, ValidationReport)}. + * Initialization-on-demand holder idiom: thread-safe with no explicit synchronization. + */ + private static Schema getXsd11DynamicDiscoverySchema() { + return Xsd11DynamicDiscoverySchemaHolder.INSTANCE; + } + + private static final class Xsd11DynamicDiscoverySchemaHolder { + private static final Schema INSTANCE; + static { + try { + INSTANCE = SchemaFactory.newInstance(XSD_1_1_NS).newSchema(); + } catch (final SAXException e) { + throw new ExceptionInInitializerError(e); + } + } + } + private XMLReader getXMLReader(final ContentHandler contentHandler, final ErrorHandler errorHandler) throws ParserConfigurationException, SAXException { diff --git a/exist-core/src/main/java/org/exist/validation/Xsd11SchemaDetection.java b/exist-core/src/main/java/org/exist/validation/Xsd11SchemaDetection.java new file mode 100644 index 00000000000..7ed36b69e2e --- /dev/null +++ b/exist-core/src/main/java/org/exist/validation/Xsd11SchemaDetection.java @@ -0,0 +1,293 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.validation; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import javax.annotation.Nullable; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.exist.resolver.ResolverFactory; +import org.xml.sax.InputSource; + +/** + * Stateless, context-free detection helpers for telling whether a schema needs XSD 1.1 to load -- + * shared between {@link org.exist.xquery.functions.validation.Jaxp} (where this logic originated) + * and store-time document validation ({@code org.exist.collections.MutableCollection}), both of + * which must route validation through an XSD 1.1-capable {@code Validator} instead of the default + * dynamic-discovery SAX pipeline, since the bundled Xerces fork's XSD 1.1 support is only wired + * into the JAXP {@code SchemaFactory}/{@code Validator} API, never into that pipeline. + * + * @see #5541 + */ +public final class Xsd11SchemaDetection { + + private static final Logger LOG = LogManager.getLogger(Xsd11SchemaDetection.class); + + private static final String XSI_NS = "http://www.w3.org/2001/XMLSchema-instance"; + private static final String XSD_VERSIONING_NS = "http://www.w3.org/2007/XMLSchema-versioning"; + + /** + * Bound on the size of {@link #CACHE} (see there for what it caches). + */ + private static final int CACHE_MAX_ENTRIES = 256; + + /** + * Cache key for {@link #CACHE}: the requesting Subject's name plus the resolved schema URI. + * Including the Subject prevents a Subject without read permission on the schema resource + * from observing a boolean populated by a different (permitted) Subject's earlier, + * permission-checked fetch -- a cache hit skips {@link #isXsd11Schema(String, String, String)}'s + * {@code openStream()} entirely, so without this the cache itself would bypass whatever + * permission check that open would otherwise perform. + */ + private record CacheKey(String subjectName, String resolvedSchemaUri) { + } + + /** + * Bounded (see {@link #CACHE_MAX_ENTRIES}), LRU-evicted cache of "does the schema at this + * resolved URI declare vc:minVersion 1.1?", so that validating many documents against the same + * schema doesn't re-fetch and re-peek it every time. Cleared by {@link #clearCache()}. + */ + private static final Cache CACHE = Caffeine.newBuilder() + .maximumSize(CACHE_MAX_ENTRIES) + .build(); + + private Xsd11SchemaDetection() { + } + + /** + * @return true if {@code message} is the "no global declaration for the root element" + * signature ({@code cvc-elt.1.a}) produced when this Xerces fork's dynamic-discovery pipeline + * meets an XSD 1.1-only schema. + */ + public static boolean isMissingElementDeclaration(@Nullable final String message) { + return message != null && message.contains("cvc-elt.1.a:"); + } + + /** + * Cheaply checks whether the instance document references a schema via {@code + * xsi:schemaLocation}/{@code xsi:noNamespaceSchemaLocation} that itself declares {@code + * vc:minVersion} containing "1.1". + * + * @param subjectName the requesting Subject's name, used to scope {@link #CACHE} (see there + * for why). + * @param peekInstance a fresh, not-yet-consumed InputSource for the instance document. + */ + public static boolean detectXsd11ViaSchemaLocation(final String subjectName, final InputSource peekInstance) { + return detectXsd11ViaSchemaLocation(subjectName, peekRootElement(peekInstance), peekInstance.getSystemId()); + } + + /** + * Same as {@link #detectXsd11ViaSchemaLocation(String, InputSource)}, but for callers that + * already peeked the root element themselves (e.g. while also checking the root namespace in + * the same StAX pass, see {@code org.exist.collections.MutableCollection}) -- avoids a second, + * redundant peek of the same document. + * + * @param rootElement the instance's root element, as already peeked by the caller via + * {@link #peekRootElement(InputSource)}, or {@code null} if that peek + * failed (treated as "no hint"). + * @param baseUri the instance's base URI (see {@link InputSource#getSystemId()}). + */ + public static boolean detectXsd11ViaSchemaLocation(final String subjectName, @Nullable final RootElementInfo rootElement, @Nullable final String baseUri) { + if (rootElement == null || baseUri == null) { + return false; + } + final Map rootAttrs = rootElement.attributes(); + + final List candidateLocations = new ArrayList<>(); + final String noNsLocation = rootAttrs.get(clark(XSI_NS, "noNamespaceSchemaLocation")); + if (noNsLocation != null) { + candidateLocations.add(noNsLocation); + } + final String schemaLocation = rootAttrs.get(clark(XSI_NS, "schemaLocation")); + if (schemaLocation != null) { + // xsi:schemaLocation is a list of "namespace location" pairs; we only need the locations. + final String[] tokens = schemaLocation.trim().split("\\s+"); + for (int i = 1; i < tokens.length; i += 2) { + candidateLocations.add(tokens[i]); + } + } + + for (final String location : candidateLocations) { + if (isXsd11Schema(subjectName, baseUri, location)) { + return true; + } + } + return false; + } + + /** + * Resolves {@code location} relative to {@code baseUri} (the same xmldb:// normalization the + * catalog mechanism uses, so this works for documents stored in the database), opens it, and + * checks whether its root element declares {@code vc:minVersion} containing "1.1". Returns + * {@code false} for any resolution/read failure -- this is a best-effort peek, not a + * substitute for the real catalog-aware resolution the actual validation pass performs. + * + *

{@code location} is the literal, attacker/document-author-controlled value of the + * instance's own {@code xsi:schemaLocation}/{@code noNamespaceSchemaLocation} hint -- if it's + * an absolute URI (e.g. {@code file:///etc/passwd}, {@code http://internal-host/...}, or even + * an absolute {@code xmldb://some-other-host:1234/db/...} naming a remote eXist instance), + * {@link URI#resolve(String)} returns it verbatim, ignoring {@code baseUri} entirely. Opening + * that unconditionally would let any caller make this (unprivileged, security-context-free) + * peek fetch arbitrary local files or issue arbitrary outbound requests using the server + * process's own OS-level/network access, regardless of the calling Subject's DB permissions -- + * this is NOT the same trust boundary as the real validation pass, which only fetches whatever + * a configured catalog/resolver permits. So only resolutions that land in the exact same + * scheme+authority (host/port) as {@code baseUri} are attempted -- i.e. genuinely relative + * locations within the instance's own origin, never a different scheme or host. Anything else + * falls through to the unchanged, already-accepted-risk default pipeline below, exactly as if + * this peek didn't exist.

+ * + *

{@code subjectName} scopes {@link #CACHE}: a cache hit skips this method's + * permission-checked {@code openStream()} entirely, so without scoping by Subject, a Subject + * without read permission on the schema resource could observe a boolean populated by a + * different (permitted) Subject's earlier fetch -- a cross-Subject information leak.

+ */ + public static boolean isXsd11Schema(final String subjectName, final String baseUri, final String location) { + try { + final URI baseUriNormalized = new URI(ResolverFactory.fixupExistCatalogUri(baseUri)); + final URI resolvedUri = baseUriNormalized.resolve(location); + if (!Objects.equals(baseUriNormalized.getScheme(), resolvedUri.getScheme()) + || !Objects.equals(baseUriNormalized.getAuthority(), resolvedUri.getAuthority())) { + LOG.debug("Refusing to peek candidate schema '{}': resolved to a different origin ('{}') than " + + "the instance's own base URI ('{}') -- leaving this to the default pipeline/catalog instead.", + location, resolvedUri, baseUriNormalized); + return false; + } + + final CacheKey cacheKey = new CacheKey(subjectName, resolvedUri.toString()); + final Boolean cached = CACHE.getIfPresent(cacheKey); + if (cached != null) { + return cached; + } + + try (final InputStream is = resolvedUri.toURL().openStream()) { + final InputSource schemaSource = new InputSource(is); + schemaSource.setSystemId(resolvedUri.toString()); + final RootElementInfo schemaRootElement = peekRootElement(schemaSource); + if (schemaRootElement == null) { + // Couldn't even parse the candidate as XML -- not a stable fact about a real + // schema, so don't cache it; let the next call retry. + return false; + } + final String minVersion = schemaRootElement.attributes().get(clark(XSD_VERSIONING_NS, "minVersion")); + final boolean result = minVersion != null && minVersion.contains("1.1"); + CACHE.put(cacheKey, result); + return result; + } + } catch (final URISyntaxException | IOException ex) { + // Not cached: this may be a transient failure (lock contention, a brief network blip + // for an xmldb:// catalog served over XML-RPC, etc); a permanently-cached false would + // wrongly keep a legitimate schema on the slower retry-after-failure path forever. + LOG.debug("Could not peek candidate schema '{}' relative to '{}': {}", location, baseUri, ex.getMessage()); + return false; + } + } + + /** + * Discards all cached {@link #isXsd11Schema(String, String, String)} results. + */ + public static void clearCache() { + CACHE.invalidateAll(); + } + + /** + * The root element's namespace URI (or {@code null} for no namespace) plus its attributes, + * keyed by Clark-notation {@code {namespace}localName} (see {@link #clark(String, String)}) -- + * as returned by {@link #peekRootElement(InputSource)}. + */ + public record RootElementInfo(@Nullable String namespaceUri, Map attributes) { + } + + /** + * Reads only as far as the root element's start tag and returns its namespace URI and + * attributes in one StAX pass -- cheaper than a full (validating) parse since StAX stops + * pulling events the moment the caller stops asking for them, and cheaper than two separate + * peeks for callers (such as {@code org.exist.collections.MutableCollection}) that need both + * pieces of information about the same root element. DTD processing and external entities are + * disabled; this reads untrusted instance documents as well as schema documents. + * + * @return the root element's info, or {@code null} if the source couldn't be read/parsed. + */ + @Nullable + public static RootElementInfo peekRootElement(final InputSource source) { + try { + final XMLInputFactory factory = hardenedXmlInputFactory(); + + final Reader characterStream = source.getCharacterStream(); + final XMLStreamReader reader = characterStream != null + ? factory.createXMLStreamReader(characterStream) + : factory.createXMLStreamReader(source.getByteStream()); + try { + while (reader.hasNext()) { + if (reader.next() == XMLStreamConstants.START_ELEMENT) { + final Map attrs = new HashMap<>(); + for (int i = 0; i < reader.getAttributeCount(); i++) { + attrs.put(clark(reader.getAttributeNamespace(i), reader.getAttributeLocalName(i)), reader.getAttributeValue(i)); + } + return new RootElementInfo(reader.getNamespaceURI(), attrs); + } + } + return null; + } finally { + reader.close(); + } + } catch (final XMLStreamException | NullPointerException ex) { + LOG.debug("Could not peek root element: {}", ex.getMessage()); + return null; + } + } + + /** + * A freshly configured StAX {@link XMLInputFactory} with DTD processing and external entities + * disabled -- shared hardening setup for {@link #peekRootElement(InputSource)} and + * {@code org.exist.collections.MutableCollection}'s equivalent root-element peek, both of which + * read untrusted instance/schema documents. + */ + public static XMLInputFactory hardenedXmlInputFactory() { + final XMLInputFactory factory = XMLInputFactory.newInstance(); + factory.setProperty(XMLInputFactory.SUPPORT_DTD, false); + factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + return factory; + } + + private static String clark(@Nullable final String namespaceUri, final String localName) { + return (namespaceUri == null ? "" : "{" + namespaceUri + "}") + localName; + } +} diff --git a/exist-core/src/main/java/org/exist/xmlrpc/RpcConnection.java b/exist-core/src/main/java/org/exist/xmlrpc/RpcConnection.java index 056bb58f6d5..14df06245cc 100644 --- a/exist-core/src/main/java/org/exist/xmlrpc/RpcConnection.java +++ b/exist-core/src/main/java/org/exist/xmlrpc/RpcConnection.java @@ -3287,7 +3287,7 @@ private boolean isValid(final XmldbURI docUri) throws EXistException { // TODO DWES reconsider try (final InputStream is = new EmbeddedInputStream(new XmldbURL(docUri))) { // Perform validation - final ValidationReport report = validator.validate(is); + final ValidationReport report = validator.validate(is, null, docUri.toString()); // Return validation result return report.isValid(); diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/transform/Transform.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/transform/Transform.java index 301ff0e7635..bf3cb0cab08 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/transform/Transform.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/transform/Transform.java @@ -222,24 +222,43 @@ private XsltExecutable compileExecutable(final Options options) throws XPathExce xsltCompiler.setParameter(new net.sf.saxon.s9api.QName(qKey.getPrefix(), qKey.getLocalPart()), value); } - xsltCompiler.setURIResolver(new URIResolution.CompileTimeURIResolver(context, fnTransform) { - @Override public Source resolve(final String href, final String base) throws TransformerException { - // Correct error from URI resolution when there is no base - try { - final URI hrefURI = URI.create(href); - if (options.resolvedStylesheetBaseURI.isEmpty() && !hrefURI.isAbsolute() && StringUtils.isEmpty(base)) { - final XPathException resolutionException = new XPathException(fnTransform, - ErrorCodes.XTSE0165, - """ - transform using a relative href,\s - using option stylesheet-text, but without stylesheet-base-uri"""); - throw new TransformerException(resolutionException); - } - } catch (final IllegalArgumentException e) { - throw new TransformerException(e); + // setResourceResolver() rather than setURIResolver() -- Saxon 12's XsltCompiler still + // accepts setURIResolver() (it wraps via ResourceResolverWrappingURIResolver), but going + // directly to the ResourceResolver API avoids that extra layer. See #350 / + // plans/catalog-dtd.plan.md. + final URIResolution.CompileTimeURIResolver delegate = new URIResolution.CompileTimeURIResolver(context, fnTransform); + xsltCompiler.setResourceResolver(request -> { + // Prefer the literal, unresolved href (relativeUri) over Saxon's already-resolved uri + // whenever it's available -- matches Saxon's own ResourceRequest.resolve() convention. + // Deliberately NOT gated on baseUri also being non-null: a relative relativeUri with a + // null baseUri is exactly the case the XTSE0165 check below exists to catch: gating on + // baseUri here would fall back to the already-resolved/absolute uri instead and let + // that case slip past the check undetected. + final String href = request.relativeUri != null ? request.relativeUri : request.uri; + final String base = request.baseUri; + if (href == null) { + // Saxon supplied neither a literal href nor a resolved uri -- nothing to resolve. + throw net.sf.saxon.trans.XPathException.makeXPathException( + new TransformerException("Could not resolve a Saxon ResourceRequest with no href (uri and relativeUri both null)")); + } + try { + final URI hrefURI = URI.create(href); + if (options.resolvedStylesheetBaseURI.isEmpty() && !hrefURI.isAbsolute() && StringUtils.isEmpty(base)) { + final XPathException resolutionException = new XPathException(fnTransform, + ErrorCodes.XTSE0165, + """ + transform using a relative href,\s + using option stylesheet-text, but without stylesheet-base-uri"""); + throw net.sf.saxon.trans.XPathException.makeXPathException(new TransformerException(resolutionException)); } - // Checked the special error case, defer to eXist resolution - return super.resolve(href, base); + } catch (final IllegalArgumentException e) { + throw net.sf.saxon.trans.XPathException.makeXPathException(new TransformerException(e)); + } + // Checked the special error case, defer to eXist resolution + try { + return delegate.resolve(href, base); + } catch (final TransformerException e) { + throw net.sf.saxon.trans.XPathException.makeXPathException(e); } }); diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/transform/URIResolution.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/transform/URIResolution.java index 5abbe73da14..1ddf644db29 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/transform/URIResolution.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/transform/URIResolution.java @@ -24,6 +24,7 @@ import org.exist.dom.persistent.NodeProxy; import org.exist.security.PermissionDeniedException; +import org.exist.util.SaxonConfiguration; import org.exist.xmldb.XmldbURI; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; @@ -34,7 +35,9 @@ import org.exist.xquery.value.Sequence; import org.exist.xquery.value.Type; import org.w3c.dom.Node; +import org.xmlresolver.Resolver; +import javax.annotation.Nullable; import javax.xml.transform.Source; import javax.xml.transform.TransformerException; import javax.xml.transform.URIResolver; @@ -74,14 +77,36 @@ public static class CompileTimeURIResolver implements URIResolver { private final XQueryContext xQueryContext; private final Expression containingExpression; + /** + * Fetched once here rather than per-href in {@link #resolveViaCatalog(String, String)} -- + * this resolver doesn't change for the lifetime of one compile, so re-fetching it from + * {@link org.exist.util.Configuration} on every {@code href} encountered while compiling a + * stylesheet (every {@code xsl:import}/{@code xsl:include}/{@code doc()}) was redundant. + */ + @Nullable + private final Resolver catalogResolver; + public CompileTimeURIResolver(XQueryContext xQueryContext, Expression containingExpression) { this.xQueryContext = xQueryContext; this.containingExpression = containingExpression; + this.catalogResolver = xQueryContext.getBroker() == null + ? null + : SaxonConfiguration.resolveCatalogResolver(xQueryContext.getBroker().getBrokerPool().getConfiguration()); } @Override public Source resolve(final String href, final String base) throws TransformerException { + // Try the system catalog (webapp/WEB-INF/catalog.xml by default) first, the same way + // XsltURIResolverHelper tries it before any network-risking fallback -- a catalog miss + // declines promptly (no fetch attempt), but resolveDocument()/DocUtils.getDocument() + // below will itself attempt a live fetch for an absolute http(s) URI, so the catalog + // must run first to avoid a slow/hanging network round-trip on every catalog hit (#350). + final Source catalogSource = resolveViaCatalog(href, base); + if (catalogSource != null) { + return catalogSource; + } + try { final AnyURIValue baseURI = new AnyURIValue(base); final AnyURIValue hrefURI = new AnyURIValue(href); @@ -96,6 +121,13 @@ public Source resolve(final String href, final String base) throws TransformerEx } } + private Source resolveViaCatalog(final String href, final String base) throws TransformerException { + if (catalogResolver == null) { + return null; + } + return catalogResolver.resolve(href, base); + } + protected Source resolveDocument(final String location) throws XPathException { return URIResolution.resolveDocument(location, xQueryContext, containingExpression); } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/validation/GrammarTooling.java b/exist-core/src/main/java/org/exist/xquery/functions/validation/GrammarTooling.java index a1705d05bfc..c720a9d636d 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/validation/GrammarTooling.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/validation/GrammarTooling.java @@ -32,6 +32,7 @@ import org.apache.xerces.xni.parser.XMLInputSource; import org.exist.Namespaces; +import org.exist.collections.MutableCollection; import org.exist.dom.QName; import org.exist.dom.memtree.MemTreeBuilder; import org.exist.dom.memtree.NodeImpl; @@ -151,6 +152,7 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) clearGrammarPool(grammarpool); Jaxp.clearXsd11DetectionCache(); + MutableCollection.clearXsd11SchemaByNamespaceCache(); final int after = countTotalNumberOfGrammar(grammarpool); LOG.debug("Remained {} grammars", after); diff --git a/exist-core/src/main/java/org/exist/xquery/functions/validation/Jaxp.java b/exist-core/src/main/java/org/exist/xquery/functions/validation/Jaxp.java index 22500bdd76e..a1d9c23d044 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/validation/Jaxp.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/validation/Jaxp.java @@ -22,30 +22,16 @@ package org.exist.xquery.functions.validation; import java.io.IOException; -import java.io.InputStream; import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; import javax.annotation.Nullable; import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamConstants; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerConfigurationException; @@ -63,7 +49,6 @@ import org.exist.dom.QName; import org.exist.dom.memtree.DocumentBuilderReceiver; import org.exist.dom.memtree.MemTreeBuilder; -import org.exist.resolver.ResolverFactory; import org.exist.resolver.XercesXmlResolverAdapter; import org.exist.storage.BrokerPool; import org.exist.util.Configuration; @@ -73,6 +58,7 @@ import org.exist.validation.GrammarPool; import org.exist.validation.ValidationContentHandler; import org.exist.validation.ValidationReport; +import org.exist.validation.Xsd11SchemaDetection; import org.exist.xquery.BasicFunction; import org.exist.xquery.Cardinality; import org.exist.xquery.FunctionSignature; @@ -109,34 +95,6 @@ public class Jaxp extends BasicFunction { private static final String XSD_1_1_NS = "http://www.w3.org/XML/XMLSchema/v1.1"; private static final String XSI_NS = "http://www.w3.org/2001/XMLSchema-instance"; - private static final String XSD_VERSIONING_NS = "http://www.w3.org/2007/XMLSchema-versioning"; - - /** - * Bound on the size of {@link #XSD11_DETECTION_CACHE} (see there for what it caches). - */ - private static final int XSD11_DETECTION_CACHE_MAX_ENTRIES = 256; - - /** - * Cache key for {@link #XSD11_DETECTION_CACHE}: the requesting Subject's name plus the - * resolved schema URI. Including the Subject prevents a Subject without read permission on - * the schema resource from observing a boolean populated by a different (permitted) - * Subject's earlier, permission-checked fetch -- a cache hit skips {@link - * #isXsd11Schema(String, String, String)}'s {@code openStream()} entirely, so without this - * the cache itself would bypass whatever permission check that open would otherwise perform. - */ - private record Xsd11DetectionCacheKey(String subjectName, String resolvedSchemaUri) { - } - - /** - * Bounded (see {@link #XSD11_DETECTION_CACHE_MAX_ENTRIES}), LRU-evicted cache of - * "does the schema at this resolved URI declare vc:minVersion 1.1?", so that validating many - * documents against the same schema doesn't re-fetch and re-peek it every time. Cleared by - * {@code validation:clear-grammar-cache()} (see {@link GrammarTooling}) alongside the Xerces - * grammar pool, so operators have one function to clear every validation-related cache. - */ - private static final Cache XSD11_DETECTION_CACHE = Caffeine.newBuilder() - .maximumSize(XSD11_DETECTION_CACHE_MAX_ENTRIES) - .build(); private static final String simpleFunctionTxt = """ Validate document by parsing $instance. Optionally \ @@ -426,7 +384,7 @@ private void setXmlReaderFeature(XMLReader xmlReader, String featureName, boolea */ static boolean isMissingElementDeclaration(final ValidationReport report) { return report.getValidationReportItemList().stream() - .anyMatch(item -> item.getMessage() != null && item.getMessage().startsWith("cvc-elt.1.a:")); + .anyMatch(item -> Xsd11SchemaDetection.isMissingElementDeclaration(item.getMessage())); } /** @@ -472,12 +430,21 @@ private record ParseTarget(ContentHandler contenthandler, @Nullable MemTreeBuild /** * Acquires a fresh, disposable {@link InputSource} for the instance and runs - * {@link #detectXsd11ViaSchemaLocation(String, InputSource)} against it. + * {@link Xsd11SchemaDetection#detectXsd11ViaSchemaLocation(String, InputSource)} against it. + *

+ * Deliberately does NOT reuse the caller's own {@code instance} {@link InputSource} (which + * would save this second acquisition): unlike the {@code EXistInputSource} subclasses {@code + * org.exist.collections.MutableCollection}'s peek relies on, {@link Shared#getInputSource} here + * returns a plain {@link InputSource} wrapping a single-use {@link java.io.InputStream} -- + * {@code getByteStream()} returns the same, already-consumed stream on a second call, not a + * fresh one. Reusing it would silently feed the real parse an exhausted stream. Investigated + * and accepted as the cost of this peek; not a candidate for the InputSource-reuse pattern used + * elsewhere. */ private boolean peekIsXsd11ViaSchemaLocation(final Sequence[] args) throws XPathException, IOException { final InputSource peekInstance = Shared.getInputSource(args[0].itemAt(0), context); try { - return detectXsd11ViaSchemaLocation(context.getSubject().getName(), peekInstance); + return Xsd11SchemaDetection.detectXsd11ViaSchemaLocation(context.getSubject().getName(), peekInstance); } finally { Shared.closeInputSource(peekInstance); } @@ -557,188 +524,21 @@ private ParseTarget retryWithXsd11ValidatorIfNeeded(final Sequence[] args, final } /** - * Best-effort, pre-parse check for whether the instance's referenced schema is XSD 1.1 - * (declares {@code vc:minVersion} containing "1.1"), so the right validation pipeline can - * be chosen up front instead of discovering the mismatch only after a failed first attempt - * (see {@link #isMissingElementDeclaration(ValidationReport)}). Resolves the schemaLocation - * hint(s) only via simple relative-URI resolution against the instance's own base URI -- - * it does NOT replicate the full catalog/entity-resolver chain used for the real parse. - * Returns {@code false} (never throws) whenever any step can't be completed; the - * retry-after-failure check in {@code eval()} remains the safety net for those cases - * (catalog-mediated locations, an unresolvable hint, etc.). - * - * @param subjectName the requesting Subject's name, used to scope {@link #XSD11_DETECTION_CACHE} - * (see there for why). - * @param peekInstance a fresh, not-yet-consumed InputSource for the same instance document. - */ - private static boolean detectXsd11ViaSchemaLocation(final String subjectName, final InputSource peekInstance) { - final Map rootAttrs = peekRootAttributes(peekInstance); - final String baseUri = peekInstance.getSystemId(); - if (rootAttrs == null || baseUri == null) { - return false; - } - - final List candidateLocations = new ArrayList<>(); - final String noNsLocation = rootAttrs.get(clark(XSI_NS, "noNamespaceSchemaLocation")); - if (noNsLocation != null) { - candidateLocations.add(noNsLocation); - } - final String schemaLocation = rootAttrs.get(clark(XSI_NS, "schemaLocation")); - if (schemaLocation != null) { - // xsi:schemaLocation is a list of "namespace location" pairs; we only need the locations. - final String[] tokens = schemaLocation.trim().split("\\s+"); - for (int i = 1; i < tokens.length; i += 2) { - candidateLocations.add(tokens[i]); - } - } - - for (final String location : candidateLocations) { - if (isXsd11Schema(subjectName, baseUri, location)) { - return true; - } - } - return false; - } - - /** - * Resolves {@code location} relative to {@code baseUri} (the same xmldb:// normalization - * the catalog mechanism uses, so this works for documents stored in the database), opens it, - * and checks whether its root element declares {@code vc:minVersion} containing "1.1". - * Returns {@code false} for any resolution/read failure -- this is a best-effort peek, not - * a substitute for the real catalog-aware resolution the actual validation pass performs. - * Package-private (not {@code private}) so {@code JaxpSchemaLocationSecurityTest}/ - * {@code JaxpXsd11DetectionCacheTest}, both in this package, can call it directly. - * - *

{@code location} is the literal, attacker/document-author-controlled value of the - * instance's own {@code xsi:schemaLocation}/{@code noNamespaceSchemaLocation} hint -- if it's - * an absolute URI (e.g. {@code file:///etc/passwd}, {@code http://internal-host/...}, or even - * an absolute {@code xmldb://some-other-host:1234/db/...} naming a remote eXist instance), - * {@link URI#resolve(String)} returns it verbatim, ignoring {@code baseUri} entirely. Opening - * that unconditionally would let any caller make this (unprivileged, security-context-free) - * peek fetch arbitrary local files or issue arbitrary outbound requests (file read, or a - * network connection -- {@code org.exist.protocolhandler.protocols.xmldb.Handler} dispatches - * to XML-RPC against whatever host an absolute {@code xmldb://} URI names) using the server - * process's own OS-level/network access, regardless of the calling Subject's DB permissions -- - * this is NOT the same trust boundary as the real validation pass, which only fetches whatever - * a configured catalog/resolver permits. So only resolutions that land in the exact same - * scheme+authority (host/port) as {@code baseUri} are attempted -- i.e. genuinely relative - * locations within the instance's own origin, never a different scheme or host. Anything else - * falls through to the unchanged, already-accepted-risk default pipeline below, exactly as if - * this peek didn't exist.

- * - *

Residual nuance, not a new gap: {@code file:} URIs have no authority component at all - * (it's always empty), so this check cannot distinguish {@code file:///a/instance.xml} from - * an absolute {@code file:///etc/passwd} hint -- both have scheme {@code file} and empty - * authority. This only matters for Java-{@link java.io.File}-backed instance items (the only - * way to get a {@code file:} base URI here), which already requires the caller to have used - * {@code util:} Java-interop functions to construct that object in the first place -- a - * separate, pre-existing privilege boundary this peek doesn't change either way.

- * - *

{@code subjectName} scopes {@link #XSD11_DETECTION_CACHE}: a cache hit skips this - * method's permission-checked {@code openStream()} entirely, so without scoping by Subject, - * a Subject without read permission on the schema resource could observe a boolean populated - * by a different (permitted) Subject's earlier fetch -- a cross-Subject information leak.

+ * Package-private delegate to {@link Xsd11SchemaDetection#isXsd11Schema(String, String, + * String)} so {@code JaxpSchemaLocationSecurityTest}/{@code JaxpXsd11DetectionCacheTest}, both + * in this package, can keep calling it directly. */ static boolean isXsd11Schema(final String subjectName, final String baseUri, final String location) { - try { - final URI baseUriNormalized = new URI(ResolverFactory.fixupExistCatalogUri(baseUri)); - final URI resolvedUri = baseUriNormalized.resolve(location); - if (!Objects.equals(baseUriNormalized.getScheme(), resolvedUri.getScheme()) - || !Objects.equals(baseUriNormalized.getAuthority(), resolvedUri.getAuthority())) { - LOG.debug("Refusing to peek candidate schema '{}': resolved to a different origin ('{}') than " + - "the instance's own base URI ('{}') -- leaving this to the default pipeline/catalog instead.", - location, resolvedUri, baseUriNormalized); - return false; - } - - final Xsd11DetectionCacheKey cacheKey = new Xsd11DetectionCacheKey(subjectName, resolvedUri.toString()); - final Boolean cached = getCachedXsd11Detection(cacheKey); - if (cached != null) { - return cached; - } - - try (final InputStream is = resolvedUri.toURL().openStream()) { - final InputSource schemaSource = new InputSource(is); - schemaSource.setSystemId(resolvedUri.toString()); - final Map schemaRootAttrs = peekRootAttributes(schemaSource); - if (schemaRootAttrs == null) { - // Couldn't even parse the candidate as XML -- not a stable fact about a real - // schema (e.g. case from #6 in detectXsd11ViaSchemaLocation finding a location - // that doesn't actually exist), so don't cache it; let the next call retry. - return false; - } - final String minVersion = schemaRootAttrs.get(clark(XSD_VERSIONING_NS, "minVersion")); - final boolean result = minVersion != null && minVersion.contains("1.1"); - cacheXsd11Detection(cacheKey, result); - return result; - } - } catch (final URISyntaxException | IOException ex) { - // Not cached: this may be a transient failure (lock contention, a brief network blip - // for an xmldb:// catalog served over XML-RPC, etc); a permanently-cached false would - // wrongly keep a legitimate schema on the slower retry-after-failure path forever. - LOG.debug("Could not peek candidate schema '{}' relative to '{}': {}", location, baseUri, ex.getMessage()); - return false; - } - } - - @Nullable - private static Boolean getCachedXsd11Detection(final Xsd11DetectionCacheKey key) { - return XSD11_DETECTION_CACHE.getIfPresent(key); - } - - private static void cacheXsd11Detection(final Xsd11DetectionCacheKey key, final boolean isXsd11) { - XSD11_DETECTION_CACHE.put(key, isXsd11); + return Xsd11SchemaDetection.isXsd11Schema(subjectName, baseUri, location); } /** - * Discards all cached {@link #isXsd11Schema(String, String, String)} results. Package-private - * so {@link GrammarTooling}'s {@code clear-grammar-cache()} can clear this alongside the - * Xerces grammar pool. + * Package-private delegate to {@link Xsd11SchemaDetection#clearCache()}, so {@link + * GrammarTooling}'s {@code clear-grammar-cache()} can clear this alongside the Xerces grammar + * pool. */ static void clearXsd11DetectionCache() { - XSD11_DETECTION_CACHE.invalidateAll(); - } - - /** - * Reads only as far as the root element's start tag and returns its attributes, keyed by - * Clark-notation {@code {namespace}localName} (see {@link #clark(String, String)}) -- cheaper - * than a full (validating) parse since StAX stops pulling events the moment the caller stops - * asking for them, and avoids the exception-as-control-flow pattern a SAX-based equivalent - * would need to abort early. DTD processing and external entities are disabled; this reads - * untrusted instance documents as well as schema documents. - * - * @return the root element's attributes, or {@code null} if the source couldn't be read/parsed. - */ - @Nullable - private static Map peekRootAttributes(final InputSource source) { - try { - final XMLInputFactory factory = XMLInputFactory.newInstance(); - factory.setProperty(XMLInputFactory.SUPPORT_DTD, false); - factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); - - final XMLStreamReader reader = factory.createXMLStreamReader(source.getByteStream()); - try { - while (reader.hasNext()) { - if (reader.next() == XMLStreamConstants.START_ELEMENT) { - final Map attrs = new HashMap<>(); - for (int i = 0; i < reader.getAttributeCount(); i++) { - attrs.put(clark(reader.getAttributeNamespace(i), reader.getAttributeLocalName(i)), reader.getAttributeValue(i)); - } - return attrs; - } - } - return null; - } finally { - reader.close(); - } - } catch (final XMLStreamException ex) { - LOG.debug("Could not peek root element attributes: {}", ex.getMessage()); - return null; - } - } - - private static String clark(@Nullable final String namespaceUri, final String localName) { - return (namespaceUri == null ? "" : "{" + namespaceUri + "}") + localName; + Xsd11SchemaDetection.clearCache(); } /** diff --git a/exist-core/src/main/java/org/exist/xslt/XsltURIResolverHelper.java b/exist-core/src/main/java/org/exist/xslt/XsltURIResolverHelper.java index e6f78fb8a40..219a76bd82c 100644 --- a/exist-core/src/main/java/org/exist/xslt/XsltURIResolverHelper.java +++ b/exist-core/src/main/java/org/exist/xslt/XsltURIResolverHelper.java @@ -24,7 +24,9 @@ import org.exist.repo.PkgXsltModuleURIResolver; import org.exist.storage.BrokerPool; import org.exist.util.EXistURISchemeURIResolver; +import org.exist.util.SaxonConfiguration; import org.exist.util.URIResolverHierarchy; +import org.xmlresolver.Resolver; import javax.annotation.Nullable; import javax.xml.transform.URIResolver; @@ -62,6 +64,15 @@ public class XsltURIResolverHelper { resolvers.add(new EXistURIResolver(brokerPool, base)); } + // System catalog (webapp/WEB-INF/catalog.xml by default, see conf.xml's entity-resolver + // config) -- lets xsl:import/xsl:include be redirected to a local resource the same way + // catalogs already work for the Xerces/JAXP validation pipeline. Tried before the default + // resolver so a catalog-redirected local copy wins over a live network fetch (#350). + final Resolver catalogResolver = SaxonConfiguration.resolveCatalogResolver(brokerPool.getConfiguration()); + if (catalogResolver != null) { + resolvers.add(catalogResolver); + } + // default resolver if (defaultResolver != null) { if (avoidSelf) { diff --git a/exist-core/src/main/resources/org/exist/util/mime-types.xml b/exist-core/src/main/resources/org/exist/util/mime-types.xml index 58ba03cf203..142f6a2318b 100644 --- a/exist-core/src/main/resources/org/exist/util/mime-types.xml +++ b/exist-core/src/main/resources/org/exist/util/mime-types.xml @@ -37,7 +37,7 @@ and then as a classpath resource in org/exist/util . ======================================================= --> - + diff --git a/exist-core/src/test/java/org/exist/util/SchemaVersionFixtureAuditTest.java b/exist-core/src/test/java/org/exist/util/SchemaVersionFixtureAuditTest.java new file mode 100644 index 00000000000..1ef1dd60bd4 --- /dev/null +++ b/exist-core/src/test/java/org/exist/util/SchemaVersionFixtureAuditTest.java @@ -0,0 +1,129 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.util; + +import org.junit.Test; +import org.w3c.dom.Document; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertTrue; + +/** + * Visibility check for test/sample fixture copies of the canonical config templates (the ones + * named e.g. {@code conf.xml} scattered across module test resources, each a hand-trimmed, + * per-module subset of the real {@code exist-distribution/.../conf.xml} -- never literal copies, + * so they can't be mechanically regenerated from canonical without destroying intentional + * per-module customization). + *

+ * None of these ~39 fixtures carry {@link SchemaVersion#ATTRIBUTE}, so none of them are checked + * for drift the way {@link SchemaVersionSyncTest} checks {@link SchemaVersion} itself. This is + * the cheaper "visibility before automation" interim step: list which fixtures are missing the + * attribute, so the gap is visible in CI rather than silent. Actually adding {@code schemaVersion} + * to all of them (via Maven resource filtering, so it can't drift once added) is a separate, + * larger follow-up -- this test does not edit any fixture. + */ +public class SchemaVersionFixtureAuditTest { + + private static final Set FIXTURE_FILE_NAMES = Set.of("conf.xml", "controller-config.xml", "collection.xconf.init"); + + /** The canonical instances themselves are not fixtures -- excluded from the scan. */ + private static final Set CANONICAL_PATHS = Set.of( + "exist-distribution/src/main/config/conf.xml", + "exist-distribution/src/main/config/collection.xconf.init", + "exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml"); + + @Test + public void reportFixturesMissingSchemaVersion() throws Exception { + final Path repoRoot = resolveRepoRoot(); + + final List fixtures = findFixtures(repoRoot); + assertTrue("expected to find test/sample fixture copies of conf.xml/controller-config.xml/" + + "collection.xconf.init under " + repoRoot + " (found none -- is repo root resolution broken?)", + !fixtures.isEmpty()); + + final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + + final List missing = new ArrayList<>(); + for (final Path fixture : fixtures) { + final Document doc = factory.newDocumentBuilder().parse(fixture.toFile()); + final String declared = doc.getDocumentElement().getAttribute(SchemaVersion.ATTRIBUTE); + if (declared == null || declared.isEmpty()) { + missing.add(repoRoot.relativize(fixture).toString()); + } + } + + // Not a hard failure (yet) -- every one of these is currently missing schemaVersion, by + // design (see class javadoc); this is the visibility step, not the enforcement step. The + // assertion just keeps the count itself from silently drifting (e.g. if a fixture + // unexpectedly starts carrying schemaVersion, or a new copy appears uninspected). + assertTrue("Found " + missing.size() + " fixture(s) without " + SchemaVersion.ATTRIBUTE + + " (expected, see class javadoc -- this is a visibility check, not enforcement): " + + missing, + missing.size() == fixtures.size()); + } + + private static List findFixtures(final Path repoRoot) throws IOException { + final List fixtures = new ArrayList<>(); + Files.walkFileTree(repoRoot, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) { + final String name = dir.getFileName() != null ? dir.getFileName().toString() : ""; + if (name.equals("target") || name.equals(".git") || name.equals(".moderne") || name.equals("node_modules")) { + return FileVisitResult.SKIP_SUBTREE; + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) { + if (FIXTURE_FILE_NAMES.contains(file.getFileName().toString())) { + final String relative = repoRoot.relativize(file).toString().replace('\\', '/'); + if (!CANONICAL_PATHS.contains(relative)) { + fixtures.add(file); + } + } + return FileVisitResult.CONTINUE; + } + }); + return fixtures; + } + + private static Path resolveRepoRoot() { + final Path base = Path.of(System.getProperty("user.dir")); + Path p = base.resolve("schema"); + if (Files.isDirectory(p)) { + return base; + } + return base.getParent(); + } +} diff --git a/exist-core/src/test/java/org/exist/util/SchemaVersionSyncTest.java b/exist-core/src/test/java/org/exist/util/SchemaVersionSyncTest.java new file mode 100644 index 00000000000..8c77bf0a902 --- /dev/null +++ b/exist-core/src/test/java/org/exist/util/SchemaVersionSyncTest.java @@ -0,0 +1,86 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.util; + +import org.junit.jupiter.api.Test; +import org.w3c.dom.Document; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The CI schema-governance workflow (.github/workflows/ci-schema-checks.yml) + * enforces that {@code xs:schema/@version} is bumped whenever a native XSD or + * its canonical template changes — but it diffs files via git path filters + * and has no visibility into Java source, so a forgotten update to the + * hand-copied constants in {@link SchemaVersion} would otherwise drift + * silently. This test closes that gap directly: it runs on every {@code mvn + * test}, independent of which files a PR happens to touch, and fails loudly + * the moment a constant disagrees with its paired XSD. + */ +public class SchemaVersionSyncTest { + + private static final Map SCHEMA_FILE_TO_CONSTANT = Map.of( + "conf.xsd", SchemaVersion.CONF, + "collection.xconf.xsd", SchemaVersion.COLLECTION_XCONF, + "descriptor.xsd", SchemaVersion.DESCRIPTOR, + "mime-types.xsd", SchemaVersion.MIME_TYPES, + "controller-config.xsd", SchemaVersion.CONTROLLER_CONFIG); + + @Test + public void schemaVersionConstantsMatchXsds() throws Exception { + final Path schemaDir = resolveSchemaDir(); + assertTrue(Files.isDirectory(schemaDir), + "schema/ directory not found at " + schemaDir + " (run from repo root: mvn test -pl exist-core)"); + + final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + + for (final Map.Entry entry : SCHEMA_FILE_TO_CONSTANT.entrySet()) { + final String fileName = entry.getKey(); + final Path xsdPath = schemaDir.resolve(fileName); + assertTrue(Files.exists(xsdPath), "Missing XSD: " + xsdPath); + + final Document doc = factory.newDocumentBuilder().parse(xsdPath.toFile()); + final String xsdVersion = doc.getDocumentElement().getAttribute("version"); + + assertEquals(xsdVersion, entry.getValue(), + "SchemaVersion.java is out of sync with schema/" + fileName + + " — update the matching constant in SchemaVersion.java" + + " whenever you bump xs:schema/@version"); + } + } + + private Path resolveSchemaDir() { + final Path base = Path.of(System.getProperty("user.dir")); + Path p = base.resolve("schema"); + if (!Files.isDirectory(p)) { + p = base.getParent().resolve("schema"); + } + return p; + } +} diff --git a/exist-core/src/test/java/org/exist/util/SchemaVersionTest.java b/exist-core/src/test/java/org/exist/util/SchemaVersionTest.java new file mode 100644 index 00000000000..eeda7739d0e --- /dev/null +++ b/exist-core/src/test/java/org/exist/util/SchemaVersionTest.java @@ -0,0 +1,145 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.util; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.config.Property; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.w3c.dom.Document; + +import javax.xml.parsers.DocumentBuilderFactory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SchemaVersionTest { + + private static final String CAPTURE_LOGGER = "org.exist.util.SchemaVersionTest.capture"; + + private CapturingAppender appender; + + @BeforeEach + public void attachAppender() { + appender = new CapturingAppender(); + appender.start(); + // The test log4j2 config sets the root logger to OFF, which would filter these events before + // any appender sees them. Install a dedicated LoggerConfig at level ALL with our capturing + // appender, mirroring DeferredFunctionCallErrorTest's pattern. + final Logger logger = (Logger) LogManager.getLogger(CAPTURE_LOGGER); + final LoggerContext ctx = logger.getContext(); + final Configuration config = ctx.getConfiguration(); + config.addAppender(appender); + final LoggerConfig loggerConfig = LoggerConfig.newBuilder() + .withLoggerName(CAPTURE_LOGGER) + .withLevel(Level.ALL) + .withAdditivity(false) + .withConfig(config) + .build(); + loggerConfig.addAppender(appender, Level.ALL, null); + config.addLogger(CAPTURE_LOGGER, loggerConfig); + ctx.updateLoggers(); + } + + @AfterEach + public void detachAppender() { + final Logger logger = (Logger) LogManager.getLogger(CAPTURE_LOGGER); + final LoggerContext ctx = logger.getContext(); + ctx.getConfiguration().removeLogger(CAPTURE_LOGGER); + ctx.updateLoggers(); + appender.stop(); + } + + @Test + public void attributeNameIsSchemaVersion() { + assertEquals("schemaVersion", SchemaVersion.ATTRIBUTE); + } + + @Test + public void logDocumentVersionAcceptsMatchingValue() throws Exception { + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); + final var root = doc.createElement("exist"); + root.setAttribute(SchemaVersion.ATTRIBUTE, SchemaVersion.CONF); + doc.appendChild(root); + + SchemaVersion.logDocumentVersion(LogManager.getLogger(CAPTURE_LOGGER), + root, SchemaVersion.CONF, "test conf.xml"); + + assertEquals(Level.DEBUG, appender.lastLevel); + assertTrue(appender.lastMessage.contains("test conf.xml")); + assertTrue(appender.lastMessage.contains(SchemaVersion.CONF)); + } + + @Test + public void logDocumentVersionAcceptsMissingAttribute() throws Exception { + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); + final var root = doc.createElement("exist"); + doc.appendChild(root); + + SchemaVersion.logDocumentVersion(LogManager.getLogger(CAPTURE_LOGGER), + root, SchemaVersion.CONF, "legacy conf.xml"); + + assertEquals(Level.DEBUG, appender.lastLevel); + assertTrue(appender.lastMessage.contains("legacy conf.xml")); + assertTrue(appender.lastMessage.contains("no " + SchemaVersion.ATTRIBUTE + " attribute")); + } + + @Test + public void logDocumentVersionWarnsOnMismatch() throws Exception { + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); + final var root = doc.createElement("exist"); + root.setAttribute(SchemaVersion.ATTRIBUTE, "0.0.1"); + doc.appendChild(root); + + SchemaVersion.logDocumentVersion(LogManager.getLogger(CAPTURE_LOGGER), + root, SchemaVersion.CONF, "outdated conf.xml"); + + assertEquals(Level.WARN, appender.lastLevel); + assertTrue(appender.lastMessage.contains("outdated conf.xml")); + assertTrue(appender.lastMessage.contains("0.0.1")); + assertTrue(appender.lastMessage.contains(SchemaVersion.CONF)); + } + + private static final class CapturingAppender extends AbstractAppender { + + private volatile Level lastLevel; + private volatile String lastMessage; + + CapturingAppender() { + super("schema-version-capture", null, null, false, Property.EMPTY_ARRAY); + } + + @Override + public void append(final LogEvent event) { + lastLevel = event.getLevel(); + lastMessage = event.getMessage().getFormattedMessage(); + } + } +} diff --git a/exist-core/src/test/java/org/exist/validation/ValidatorXsd11Test.java b/exist-core/src/test/java/org/exist/validation/ValidatorXsd11Test.java new file mode 100644 index 00000000000..66b30c50c4b --- /dev/null +++ b/exist-core/src/test/java/org/exist/validation/ValidatorXsd11Test.java @@ -0,0 +1,130 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.validation; + +import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream; +import org.exist.security.AuthenticationException; +import org.exist.storage.BrokerPool; +import org.exist.test.ExistEmbeddedServer; +import org.junit.ClassRule; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.exist.TestUtils.ADMIN_DB_PWD; +import static org.exist.TestUtils.ADMIN_DB_USER; +import static org.exist.util.PropertiesBuilder.propertiesBuilder; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * {@link Validator#validateParse(InputStream, String, String)} (the org.exist.xmlrpc.RpcConnection + * {@code isValid()} XML-RPC method's underlying implementation) builds its own validating SAX + * {@link org.xml.sax.XMLReader} -- the same XSD-1.0-only dynamic-discovery pipeline {@code + * org.exist.collections.MutableCollection}'s store-time validation used to be stuck with, before + * an up-front {@code xsi:schemaLocation} peek + XSD 1.1 {@link javax.xml.validation.ValidatorHandler} + * routing was added here too. + * + * @see #6189 + */ +public class ValidatorXsd11Test { + + @ClassRule + public static final ExistEmbeddedServer existEmbeddedServer = new ExistEmbeddedServer( + propertiesBuilder().build(), true, true); + + private static final String XSD_1_1_ONLY_SCHEMA = """ + + + + + + + + + + + + """; + + private static final String INSTANCE_TEMPLATE = """ + + %d + %d + + """; + + @Test + public void conformingInstanceAgainstXsd11SchemaViaLocationHintIsValid() throws Exception { + final Path tempDir = Files.createTempDirectory("validator-xsd11-conform-test"); + try { + Files.writeString(tempDir.resolve("schema.xsd"), XSD_1_1_ONLY_SCHEMA, UTF_8); + final String documentBaseUri = tempDir.resolve("instance.xml").toUri().toString(); + final String instance = INSTANCE_TEMPLATE.formatted(1, 2); + + final ValidationReport report = validate(instance, documentBaseUri); + + assertTrue("conforming instance against an XSD-1.1-only schema (via schemaLocation hint) should be valid: " + + describeFailure(report), report.isValid()); + } finally { + Files.deleteIfExists(tempDir.resolve("schema.xsd")); + Files.deleteIfExists(tempDir); + } + } + + @Test + public void violatingInstanceAgainstXsd11SchemaViaLocationHintIsNotValid() throws Exception { + final Path tempDir = Files.createTempDirectory("validator-xsd11-violate-test"); + try { + Files.writeString(tempDir.resolve("schema.xsd"), XSD_1_1_ONLY_SCHEMA, UTF_8); + final String documentBaseUri = tempDir.resolve("instance.xml").toUri().toString(); + // value2 (1) is not greater than value1 (2) -- violates the xs:assert. + final String instance = INSTANCE_TEMPLATE.formatted(2, 1); + + final ValidationReport report = validate(instance, documentBaseUri); + + assertFalse("instance violating the xs:assert should not be valid", report.isValid()); + } finally { + Files.deleteIfExists(tempDir.resolve("schema.xsd")); + Files.deleteIfExists(tempDir); + } + } + + private static ValidationReport validate(final String instance, final String documentBaseUri) throws IOException, AuthenticationException { + final BrokerPool pool = existEmbeddedServer.getBrokerPool(); + final Validator validator = new Validator(pool, + pool.getSecurityManager().authenticate(ADMIN_DB_USER, ADMIN_DB_PWD)); + try (final InputStream is = new UnsynchronizedByteArrayInputStream(instance.getBytes(UTF_8))) { + return validator.validate(is, null, documentBaseUri); + } + } + + private static String describeFailure(final ValidationReport report) { + return report.getThrowable() != null ? report.getThrowable().getMessage() : String.join("; ", report.getValidationReportArray()); + } +} diff --git a/exist-core/src/test/java/org/exist/validation/Xsd11StoreTimeValidationTest.java b/exist-core/src/test/java/org/exist/validation/Xsd11StoreTimeValidationTest.java new file mode 100644 index 00000000000..0933e07b67c --- /dev/null +++ b/exist-core/src/test/java/org/exist/validation/Xsd11StoreTimeValidationTest.java @@ -0,0 +1,278 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.validation; + +import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream; +import org.exist.collections.Collection; +import org.exist.storage.BrokerPool; +import org.exist.storage.DBBroker; +import org.exist.dom.persistent.LockedDocument; +import org.exist.storage.lock.Lock; +import org.exist.storage.txn.TransactionManager; +import org.exist.storage.txn.Txn; +import org.exist.test.ExistEmbeddedServer; +import org.exist.util.XMLReaderObjectFactory; +import org.exist.xmldb.XmldbURI; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.w3c.dom.CDATASection; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.exist.TestUtils.ADMIN_DB_PWD; +import static org.exist.TestUtils.ADMIN_DB_USER; +import static org.exist.TestUtils.GUEST_DB_USER; +import static org.exist.util.PropertiesBuilder.propertiesBuilder; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * At-store-time validation (the path {@code org.exist.collections.MutableCollection} drives for + * {@code }) must work for a genuinely XSD-1.1-only, user-authored + * schema -- not just the W3C XSD 1.1 meta-schema special case the original regression (storing a + * {@code .xsd} document itself) surfaced. The bundled Xerces fork's XSD 1.1 support is only wired + * into the JAXP {@code SchemaFactory}/{@code Validator} API, never into the default + * dynamic-discovery SAX pipeline, so both of these must be detected and routed up front: + * + *

    + *
  1. Storing the schema document itself (root element in the W3C XML Schema namespace, no + * {@code schemaLocation} hint at all) -- exercises the namespace-resolution path.
    2. + *
  2. Storing an instance that references that schema via {@code + * xsi:noNamespaceSchemaLocation} -- exercises the schemaLocation-hint path shared with {@code + * validation:jaxp()} (see {@link Xsd11SchemaDetection}).
    3. + *
+ * + *

Inserts via {@code xmldb:exist://} URL upload ({@link TestTools#insertDocumentToURL}, the + * same mechanism {@link DatabaseInsertResourcesWithValidationTest} uses), not the XQuery {@code + * xmldb:store()} function: a document constructed/serialized inline within an XQuery has no + * meaningful base URI by the time it reaches {@code MutableCollection} (confirmed empirically -- + * its {@code InputSource} system ID is {@code null}), so a relative or same-origin-absolute + * {@code schemaLocation} hint could never be resolved that way regardless of this fix. The + * URL-upload path writes through a real temp file ({@code FileInputSource}), which does carry a + * real {@code file:} system ID -- the same precondition relative/same-origin {@code + * schemaLocation} resolution already needed for any pre-existing (XSD 1.0, DTD) store-time + * validation against a relative hint.

+ * + * @see #5541 + */ +public class Xsd11StoreTimeValidationTest { + + @ClassRule + public static final ExistEmbeddedServer existEmbeddedServer = new ExistEmbeddedServer( + propertiesBuilder() + .set(XMLReaderObjectFactory.PROPERTY_VALIDATION_MODE, "yes") + .build(), + true, + true); + + private static final String TEST_COLLECTION_URI = "/db/xsd11storetimevalidation"; + + private static final String XSD_1_1_ONLY_SCHEMA = """ + + + + + + + + + + + + """; + + private static final String INSTANCE_TEMPLATE = """ + + %d + %d + + """; + + private static final String COMMENT_TEXT = " a comment "; + + private static final String INSTANCE_WITH_COMMENT_AND_CDATA_TEMPLATE = """ + + %d + + """; + + private static final String XCONF_YES = """ + + + + """; + + @BeforeClass + public static void createTestCollection() throws Exception { + final BrokerPool pool = existEmbeddedServer.getBrokerPool(); + final TransactionManager transact = pool.getTransactionManager(); + try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().authenticate(ADMIN_DB_USER, ADMIN_DB_PWD))); + final Txn txn = transact.beginTransaction()) { + final Collection testCollection = broker.getOrCreateCollection(txn, XmldbURI.create(TEST_COLLECTION_URI)); + testCollection.getPermissions().setOwner(GUEST_DB_USER); + broker.saveCollection(txn, testCollection); + + final Collection configCollection = broker.getOrCreateCollection(txn, + XmldbURI.create("/db/system/config" + TEST_COLLECTION_URI)); + configCollection.getPermissions().setOwner(GUEST_DB_USER); + broker.saveCollection(txn, configCollection); + + transact.commit(txn); + } + + // A leading-CollectionConfiguration-without-an-explicit- element resolves to + // VALIDATION_SETTING.UNKNOWN ("maybe() == false"), which *disables* validation regardless + // of any global validation.mode default -- an explicit collection.xconf is required. + TestTools.insertDocumentToURL( + new UnsynchronizedByteArrayInputStream(XCONF_YES.getBytes(UTF_8)), + "xmldb:exist:///db/system/config" + TEST_COLLECTION_URI + "/collection.xconf"); + } + + @AfterClass + public static void removeTestCollection() throws Exception { + final BrokerPool pool = existEmbeddedServer.getBrokerPool(); + final TransactionManager transact = pool.getTransactionManager(); + try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().authenticate(ADMIN_DB_USER, ADMIN_DB_PWD))); + final Txn txn = transact.beginTransaction()) { + final Collection testCollection = broker.getOrCreateCollection(txn, XmldbURI.create(TEST_COLLECTION_URI)); + broker.removeCollection(txn, testCollection); + transact.commit(txn); + } + } + + @Test + public void xsd11SchemaDocumentItselfStoresUnderValidation() { + // Storing the schema document itself validates it against the W3C meta-schema, purely by + // namespace (no schemaLocation hint at all) -- exercises resolveXsd11SchemaForNamespace(). + try { + TestTools.insertDocumentToURL( + new UnsynchronizedByteArrayInputStream(XSD_1_1_ONLY_SCHEMA.getBytes(UTF_8)), + "xmldb:exist://" + TEST_COLLECTION_URI + "/schema-self.xsd"); + } catch (final IOException e) { + fail("storing XSD 1.1 schema should not throw: " + e.getMessage()); + } + } + + @Test + public void conformingInstanceAgainstXsd11SchemaViaLocationHintStores() throws Exception { + final Path schema = writeTempSchema("xsd11store-conform-test"); + try { + // The instance's own xsi:noNamespaceSchemaLocation hint resolves to an XSD 1.1-only + // schema -- exercises Xsd11SchemaDetection.detectXsd11ViaSchemaLocation() and the + // dynamic discovery XSD 1.1 Validator. + final String instance = INSTANCE_TEMPLATE.formatted(schema.toUri(), 1, 2); + TestTools.insertDocumentToURL( + new UnsynchronizedByteArrayInputStream(instance.getBytes(UTF_8)), + "xmldb:exist://" + TEST_COLLECTION_URI + "/instance-conform.xml"); + } catch (final Exception e) { + fail("conforming instance should store without exception: " + e.getMessage()); + } finally { + Files.deleteIfExists(schema); + Files.deleteIfExists(schema.getParent()); + } + } + + @Test + public void commentAndCdataSurviveXsd11StoreTimeValidation() throws Exception { + // The XSD 1.1 store-time path drives a ValidatorHandler via xmlReader1.parse(source) + // rather than Schema.newValidator()'s validate(Source, SAXResult) precisely so that + // comments/CDATA are not silently dropped -- a SAXResult has no lexical-handler hook. + // Confirms that fix concretely, not just "store doesn't throw". + final Path schema = writeTempSchema("xsd11store-lexical-test"); + try { + final String instance = INSTANCE_WITH_COMMENT_AND_CDATA_TEMPLATE.formatted(schema.toUri(), COMMENT_TEXT, 1, 2); + TestTools.insertDocumentToURL( + new UnsynchronizedByteArrayInputStream(instance.getBytes(UTF_8)), + "xmldb:exist://" + TEST_COLLECTION_URI + "/instance-lexical.xml"); + + final BrokerPool pool = existEmbeddedServer.getBrokerPool(); + try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().authenticate(ADMIN_DB_USER, ADMIN_DB_PWD))); + final LockedDocument lockedDocument = broker.getXMLResource( + XmldbURI.create(TEST_COLLECTION_URI + "/instance-lexical.xml"), Lock.LockMode.READ_LOCK)) { + assertNotNull("stored document should be retrievable", lockedDocument); + + final Document document = lockedDocument.getDocument(); + final Element root = document.getDocumentElement(); + final NodeList rootChildren = root.getChildNodes(); + + final Node commentNode = rootChildren.item(0); + assertEquals("comment should survive store-time XSD 1.1 validation", + Node.COMMENT_NODE, commentNode.getNodeType()); + assertEquals(COMMENT_TEXT, commentNode.getNodeValue()); + + final Node value1 = rootChildren.item(1); + final Node value1Child = value1.getFirstChild(); + assertEquals("CDATA section should survive store-time XSD 1.1 validation as a CDATASection node, not plain text", + Node.CDATA_SECTION_NODE, value1Child.getNodeType()); + assertEquals("1", ((CDATASection) value1Child).getData()); + } + } finally { + Files.deleteIfExists(schema); + Files.deleteIfExists(schema.getParent()); + } + } + + @Test + public void violatingInstanceAgainstXsd11SchemaViaLocationHintFails() throws Exception { + final Path schema = writeTempSchema("xsd11store-violate-test"); + try { + final String instance = INSTANCE_TEMPLATE.formatted(schema.toUri(), 2, 1); + try { + TestTools.insertDocumentToURL( + new UnsynchronizedByteArrayInputStream(instance.getBytes(UTF_8)), + "xmldb:exist://" + TEST_COLLECTION_URI + "/instance-violate.xml"); + fail("should have failed: value2 is not greater than value1"); + } catch (final IOException ex) { + final String msg = ex.getCause() != null ? ex.getCause().getMessage() : ex.getMessage(); + assertTrue("expected an xs:assert violation message, got: " + msg, + msg.contains("cvc-assertion") || msg.contains("value2 gt value1")); + } + } finally { + Files.deleteIfExists(schema); + Files.deleteIfExists(schema.getParent()); + } + } + + private static Path writeTempSchema(final String tempDirPrefix) throws Exception { + final Path tempDir = Files.createTempDirectory(tempDirPrefix); + final Path schema = tempDir.resolve("schema.xsd"); + Files.writeString(schema, XSD_1_1_ONLY_SCHEMA, UTF_8); + return schema; + } +} diff --git a/exist-core/src/test/java/xquery/xquery3/XQuery3Tests.java b/exist-core/src/test/java/xquery/xquery3/XQuery3Tests.java index cdf0ba44d4d..1363a2bd93a 100644 --- a/exist-core/src/test/java/xquery/xquery3/XQuery3Tests.java +++ b/exist-core/src/test/java/xquery/xquery3/XQuery3Tests.java @@ -28,6 +28,7 @@ @XSuite.XSuiteFiles({ "src/test/xquery/xquery3", "src/test/xquery/xquery3/transform", + "src/test/xquery/transform", // To add an individual test or only run a specific set of tests - //"src/test/xquery/xquery3/serialize.xql", }) diff --git a/exist-core/src/test/resources/org/exist/validation/catalog.xml b/exist-core/src/test/resources/org/exist/validation/catalog.xml index 5454fbdb01a..475c83e1a6a 100644 --- a/exist-core/src/test/resources/org/exist/validation/catalog.xml +++ b/exist-core/src/test/resources/org/exist/validation/catalog.xml @@ -184,7 +184,12 @@ - + + + + diff --git a/exist-core/src/test/resources/org/exist/validation/entities/XMLSchema.dtd b/exist-core/src/test/resources/org/exist/validation/entities/XMLSchema.dtd new file mode 100644 index 00000000000..64aa2d97019 --- /dev/null +++ b/exist-core/src/test/resources/org/exist/validation/entities/XMLSchema.dtd @@ -0,0 +1,513 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +%xs-datatypes; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/exist-core/src/test/resources/org/exist/validation/entities/XMLSchema.xsd b/exist-core/src/test/resources/org/exist/validation/entities/XMLSchema.xsd index 575975b412e..21c707cd4a4 100644 --- a/exist-core/src/test/resources/org/exist/validation/entities/XMLSchema.xsd +++ b/exist-core/src/test/resources/org/exist/validation/entities/XMLSchema.xsd @@ -1,2163 +1,1558 @@ - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]> + + - Part 1 version: Id: structures.xsd,v 1.2 2004/01/15 11:34:25 ht Exp - Part 2 version: Id: datatypes.xsd,v 1.3 2004/01/23 18:11:13 ht Exp + Part 1 version: structures.xsd (rec-20120405) + Part 2 version: datatypes.xsd (rec-20120405) - - + + The schema corresponding to this document is normative, with respect to the syntactic constraints it expresses in the - XML Schema language. The documentation (within <documentation> elements) + XML Schema Definition Language. The documentation (within 'documentation' elements) below, is not normative, but rather highlights important aspects of - the W3C Recommendation of which this is a part - + the W3C Recommendation of which this is a part. - - + See below (at the bottom of this document) for information about + the revision and namespace-versioning policy governing this + schema document. + + + + + The simpleType element and all of its members are defined - towards the end of this schema document + towards the end of this schema document. - - - - + + + Get access to the xml: attribute groups for xml:lang as declared on 'schema' and 'documentation' below - - - - - - + + + + + This type is extended by almost all schema types to allow attributes from other namespaces to be added to user schemas. - - - - - - - - - - - + + + + + + + + + + This type is extended by all types which allow annotation - other than <schema> itself + other than <schema> itself - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + This group is for the elements which occur freely at the top level of schemas. All of their types are based on the "annotated" type by extension. - - - - - - - - - - - - + + + + + + + + + + + This group is for the elements which can self-redefine (see <redefine> below). - - - - - - - - - - - - + + + + + + + + + + + A utility type, not for public use - - - - - - - - - - + + + + + + + + + A utility type, not for public use - - - - - - - - - - + + + + + + + + + A utility type, not for public use - + #all or (possibly empty) subset of {extension, restriction} - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + A utility type, not for public use - - - - - - - - - + + + + + + + + - - + + A utility type, not for public use - + #all or (possibly empty) subset of {extension, restriction, list, union} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - for maxOccurs - - - - - - - - - - - - for all particles - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + for maxOccurs + + + + + + + + + + + + + for all particles + + + + + + + for element, group and attributeGroup, - which both define and reference - - - - - - - - 'complexType' uses this - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + which both define and reference + + + + + + + + 'complexType' uses this + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + - - + + This branch is short for <complexContent> <restriction base="xs:anyType"> ... </restriction> </complexContent> - - - + + + + + - - - - - - - - - - - Will be restricted to required or forbidden - - - - - + + + + + + + + + + Will be restricted to required or prohibited + + + + + Not allowed if simpleContent child is chosen. - May be overriden by setting on complexContent child. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - This choice is added simply to - make this a valid restriction per the REC - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Overrides any setting on complexType parent. - - - - + May be overridden by setting on complexContent child. + + + + + + + + - - - - - - - - - - This choice is added simply to - make this a valid restriction per the REC - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This choice is added simply to + make this a valid restriction per the REC + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Overrides any setting on complexType parent. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This choice is added simply to + make this a valid restriction per the REC + + + + + + + + + + + + + + + + No typeDefParticle group reference - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - + + A utility type, not for public use - + #all or (possibly empty) subset of {substitution, extension, restriction} - - - - - - - - - + + + + + + + - - - - - + + + + + + + + + - - - + - - - - + + + The element element can be used either at the top level to define an element-type binding globally, or within a content model to either reference a globally-defined element or type or declare an element-type binding locally. The ref form is not allowed at the top level. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This type is used for 'alternative' elements. + + + + + + + + + + + + + + + + + group type for explicit groups, named top-level groups and group references - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - group type for the three kinds of group - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - This choice with min/max is here to - avoid a pblm with the Elt:All/Choice/Seq - Particle derivation constraint - - - - - - - - - - restricted max/min - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - Only elements allowed inside - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + - - - - - simple type for the value of the 'namespace' attr of - 'any' and 'anyAttribute' - - - - Value is - ##any - - any non-conflicting WFXML/attribute at all - - ##other - - any non-conflicting WFXML/attribute from - namespace other than targetNS - - ##local - - any unqualified non-conflicting WFXML/attribute - - one or - - any non-conflicting WFXML/attribute from - more URI the listed namespaces - references - (space separated) - - ##targetNamespace or ##local may appear in the above list, to - refer to the targetNamespace of the enclosing - schema or an absent targetNamespace respectively - - - - - - A utility type, not for public use - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + group type for the three kinds of group + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - A subset of XPath expressions for use -in selectors - A utility type, not for public -use - - - - The following pattern is intended to allow XPath - expressions per the following EBNF: - Selector ::= Path ( '|' Path )* - Path ::= ('.//')? Step ( '/' Step )* - Step ::= '.' | NameTest - NameTest ::= QName | '*' | NCName ':' '*' - child:: is also allowed - - - - - - - - - - - - - - - - - - - - - - - A subset of XPath expressions for use -in fields - A utility type, not for public -use - - + + + + - The following pattern is intended to allow XPath - expressions per the same EBNF as for selector, - with the following change: - Path ::= ('.//')? ( Step '/' )* ( Step | '@' NameTest ) - + This choice with min/max is here to + avoid a pblm with the Elt:All/Choice/Seq + Particle derivation constraint - - - - - - - - - - - - - - - - - - - - - - - - - The three kinds of identity constraints, all with - type of or derived from 'keybase'. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - A utility type, not for public use - - A public identifier, per ISO 8879 - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + Only elements allowed inside + - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - notations for use within XML Schema schemas - - - - - - - - - Not the real urType, but as close an approximation as we can - get in the XML representation - - - - - - - - - - First the built-in primitive datatypes. These definitions are for - information only, the real built-in definitions are magic. - - - - For each built-in datatype in this schema (both primitive and - derived) can be uniquely addressed via a URI constructed - as follows: - 1) the base URI is the URI of the XML Schema namespace - 2) the fragment identifier is the name of the datatype - - For example, to address the int datatype, the URI is: - - http://www.w3.org/2001/XMLSchema#int - - Additionally, each facet definition element can be uniquely - addressed via a URI constructed as follows: - 1) the base URI is the URI of the XML Schema namespace - 2) the fragment identifier is the name of the facet - - For example, to address the maxInclusive facet, the URI is: - - http://www.w3.org/2001/XMLSchema#maxInclusive - - Additionally, each facet usage in a built-in datatype definition - can be uniquely addressed via a URI constructed as follows: - 1) the base URI is the URI of the XML Schema namespace - 2) the fragment identifier is the name of the datatype, followed - by a period (".") followed by the name of the facet - - For example, to address the usage of the maxInclusive facet in - the definition of int, the URI is: - - http://www.w3.org/2001/XMLSchema#int.maxInclusive - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - + + + + source="../structures/structures.html#element-choice"/> - - - - - - - - - - - - - - - - - - - - - - + + + + source="../structures/structures.html#element-sequence"/> - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + simple type for the value of the 'namespace' attr of + 'any' and 'anyAttribute' + + + + Value is + ##any - - any non-conflicting WFXML/attribute at all - - - - - - - - - - - - - - - - - - - - - + ##other - - any non-conflicting WFXML/attribute from + namespace other than targetNS - - - - - - - - - - - - - - - - - - - - - + ##local - - any unqualified non-conflicting WFXML/attribute - - - - - - - - - - - - - - - - - - - - - + one or - - any non-conflicting WFXML/attribute from + more URI the listed namespaces + references + (space separated) - - - - - - - - - - - - - - - - + ##targetNamespace or ##local may appear in the above list, to + refer to the targetNamespace of the enclosing + schema or an absent targetNamespace respectively + + + + + A utility type, not for public use - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + A utility type, not for public use - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + A utility type, not for public use - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + A utility type, not for public use + - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + - NOTATION cannot be used directly in a schema; rather a type - must be derived from it by specifying at least one enumeration - facet whose value is the name of a NOTATION declared in the - schema. + A utility type, not for public use - - - + + + + + + + + + + + - - - - Now the derived primitive types - - - - + + + + + + + + + + + + + source="../structures/structures.html#element-attribute"/> - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + source="../structures/structures.html#element-attributeGroup"/> - - - - - - + + + source="../structures/structures.html#element-include"/> - - - - - pattern specifies the content of section 2.12 of XML 1.0e2 - and RFC 3066 (Revised version of RFC 1766). - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + source="../structures/structures.html#element-redefine"/> - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + source="../structures/structures.html#element-override"/> - - - - - - - - - + + + + + + + + + + + + + + + source="../structures/structures.html#element-import"/> - - - - - pattern matches production 7 from the XML spec - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + source="../structures/structures.html#element-selector"/> - - - - - - - - - + + + + + + + A subset of XPath expressions for use +in selectors + A utility type, not for public +use + + + + + + + + + + + + + + + + + + + + + A subset of XPath expressions for use +in fields + A utility type, not for public +use + + + + + + + + + + + + + + + + + + + + + + + - + The three kinds of identity constraints, all with + type of or derived from 'keybase'. + - - - - - pattern matches production 5 from the XML spec - - - - - - - + + + + + + + + source="../structures/structures.html#element-unique"/> - - - - - pattern matches production 4 from the Namespaces in XML spec - - - - - - - + + - + - - - - + + + source="../structures/structures.html#element-keyref"/> - - - - + + + + + + + + + + source="../structures/structures.html#element-notation"/> - - - - + + + + + + + + + + + - + + A utility type, not for public use + + A public identifier, per ISO 8879 - - - - + - - + + source="../structures/structures.html#element-appinfo"/> - - - - - - + + + + + + + + + + source="../structures/structures.html#element-documentation"/> - - - - - - + + + + + + + + + + - - - - + source="../structures/structures.html#element-annotation"/> - - - - - - - + + + + + + + + + + + + + + + notations for use within schema documents + + + + - + + Not the real urType, but as close an approximation as we can + get in the XML representation - - - - - + + + + + - - - - - - - - - + + + In keeping with the XML Schema WG's standard versioning policy, + the material in this schema document will persist at the URI + http://www.w3.org/2012/04/XMLSchema.xsd. - - - - - - - - - + At the date of issue it can also be found at the URI + http://www.w3.org/2009/XMLSchema/XMLSchema.xsd. - - - - - - - - + The schema document at that URI may however change in the future, + in order to remain compatible with the latest version of XSD + and its namespace. In other words, if XSD or the XML Schema + namespace change, the version of this document at + http://www.w3.org/2009/XMLSchema/XMLSchema.xsd will change accordingly; + the version at http://www.w3.org/2012/04/XMLSchema.xsd will not change. - - - - - - - - - - - - + Previous dated (and unchanging) versions of this schema document + include: - - - - - - - - + http://www.w3.org/2012/01/XMLSchema.xsd + (XSD 1.1 Proposed Recommendation) - - - - - - - - + http://www.w3.org/2011/07/XMLSchema.xsd + (XSD 1.1 Candidate Recommendation) - - - - - - - - + http://www.w3.org/2009/04/XMLSchema.xsd + (XSD 1.1 Candidate Recommendation) - - - - - - - - + http://www.w3.org/2004/10/XMLSchema.xsd + (XSD 1.0 Recommendation, Second Edition) - - - - A utility type, not for public use + http://www.w3.org/2001/05/XMLSchema.xsd + (XSD 1.0 Recommendation, First Edition) + + + - - - - - - - - - - - - - - - - - - - #all or (possibly empty) subset of {restriction, union, list} - - + + + + + A utility type, not for public use - - - - - + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + #all or (possibly empty) subset of {restriction, extension, union, list} + + + A utility type, not for public use + + + + + + + + + + + + + + + + + + + + + @@ -2173,7 +1568,6 @@ use - @@ -2181,19 +1575,17 @@ use - + Required at the top level - + - @@ -2209,72 +1601,59 @@ use - + - + source="http://www.w3.org/TR/xmlschema11-2/#element-simpleType"/> + + + + + + An abstract element, representing facets in general. + The facets defined by this spec are substitutable for + this element, and implementation-defined facets should + also name this as a substitution-group head. + - - - - - We should use a substitution group for facets, but - that's ruled out because it would allow users to - add their own, which we're not ready for yet. - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - + + + base attribute and simpleType child are mutually exclusive, but one or other is required - - + + - - - - + + + itemType attribute and simpleType child are mutually exclusive, but one or other is required @@ -2283,19 +1662,18 @@ use + minOccurs="0"/> - - - - + + + memberTypes attribute must be non-empty or there must be at least one simpleType child @@ -2304,7 +1682,7 @@ use + minOccurs="0" maxOccurs="unbounded"/> @@ -2315,71 +1693,88 @@ use - - + - - - - - - - - - - - - - - + + + + + + + + + + + + + source="http://www.w3.org/TR/xmlschema11-2/#element-minExclusive"/> - + + source="http://www.w3.org/TR/xmlschema11-2/#element-minInclusive"/> - - + + source="http://www.w3.org/TR/xmlschema11-2/#element-maxExclusive"/> - + + source="http://www.w3.org/TR/xmlschema11-2/#element-maxInclusive"/> - - - - - - + + + + + + + + + + + + + + + + + - + + source="http://www.w3.org/TR/xmlschema11-2/#element-totalDigits"/> @@ -2388,48 +1783,56 @@ use - + - + + source="http://www.w3.org/TR/xmlschema11-2/#element-fractionDigits"/> - + + source="http://www.w3.org/TR/xmlschema11-2/#element-length"/> - + + source="http://www.w3.org/TR/xmlschema11-2/#element-minLength"/> - + + source="http://www.w3.org/TR/xmlschema11-2/#element-maxLength"/> - - + + source="http://www.w3.org/TR/xmlschema11-2/#element-enumeration"/> - - + + source="http://www.w3.org/TR/xmlschema11-2/#element-whiteSpace"/> @@ -2446,16 +1849,16 @@ use - + - - + + source="http://www.w3.org/TR/xmlschema11-2/#element-pattern"/> @@ -2463,11 +1866,85 @@ use - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In keeping with the XML Schema WG's standard versioning policy, + this schema document will persist at the URI + http://www.w3.org/2012/04/datatypes.xsd. + + At the date of issue it can also be found at the URI + http://www.w3.org/2009/XMLSchema/datatypes.xsd. + + The schema document at that URI may however change in the future, + in order to remain compatible with the latest version of XSD + and its namespace. In other words, if XSD or the XML Schema + namespace change, the version of this document at + http://www.w3.org/2009/XMLSchema/datatypes.xsd will change accordingly; + the version at http://www.w3.org/2012/04/datatypes.xsd will not change. + + Previous dated (and unchanging) versions of this schema document + include: + + http://www.w3.org/2012/01/datatypes.xsd + (XSD 1.1 Proposed Recommendation) + + http://www.w3.org/2011/07/datatypes.xsd + (XSD 1.1 Candidate Recommendation) + + http://www.w3.org/2009/04/datatypes.xsd + (XSD 1.1 Candidate Recommendation) + + http://www.w3.org/2004/10/datatypes.xsd + (XSD 1.0 Recommendation, Second Edition) + + http://www.w3.org/2001/05/datatypes.xsd + (XSD 1.0 Recommendation, First Edition) + + + + + + diff --git a/exist-core/src/test/resources/org/exist/validation/entities/datatypes.dtd b/exist-core/src/test/resources/org/exist/validation/entities/datatypes.dtd new file mode 100644 index 00000000000..f9352bae1c4 --- /dev/null +++ b/exist-core/src/test/resources/org/exist/validation/entities/datatypes.dtd @@ -0,0 +1,222 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/exist-core/src/test/resources/org/exist/validation/entities/transform-catalog-test-lib.xsl b/exist-core/src/test/resources/org/exist/validation/entities/transform-catalog-test-lib.xsl new file mode 100644 index 00000000000..11a54ce73a6 --- /dev/null +++ b/exist-core/src/test/resources/org/exist/validation/entities/transform-catalog-test-lib.xsl @@ -0,0 +1,6 @@ + + + hello + + diff --git a/exist-core/src/test/resources/org/exist/validation/entities/xml.xsd b/exist-core/src/test/resources/org/exist/validation/entities/xml.xsd index de11f7e07b2..aea7d0db0a4 100644 --- a/exist-core/src/test/resources/org/exist/validation/entities/xml.xsd +++ b/exist-core/src/test/resources/org/exist/validation/entities/xml.xsd @@ -1,107 +1,107 @@ - - - - See http://www.w3.org/XML/1998/namespace.html and - http://www.w3.org/TR/REC-xml for information about this namespace. - - This schema document describes the XML namespace, in a form - suitable for import by other schema documents. - - Note that local names in this namespace are intended to be defined - only by the World Wide Web Consortium or its subgroups. The - following names are currently defined in this namespace and should - not be used with conflicting semantics by any Working Group, - specification, or document instance: - - base (as an attribute name): denotes an attribute whose value - provides a URI to be used as the base for interpreting any - relative URIs in the scope of the element on which it - appears; its value is inherited. This name is reserved - by virtue of its definition in the XML Base specification. - - id (as an attribute name): denotes an attribute whose value - should be interpreted as if declared to be of type ID. - The xml:id specification is not yet a W3C Recommendation, - but this attribute is included here to facilitate experimentation - with the mechanisms it proposes. Note that it is _not_ included - in the specialAttrs attribute group. - - lang (as an attribute name): denotes an attribute whose value - is a language code for the natural language of the content of - any element; its value is inherited. This name is reserved - by virtue of its definition in the XML specification. - - space (as an attribute name): denotes an attribute whose - value is a keyword indicating what whitespace processing - discipline is intended for the content of the element; its - value is inherited. This name is reserved by virtue of its - definition in the XML specification. - - Father (in any context at all): denotes Jon Bosak, the chair of - the original XML Working Group. This name is reserved by - the following decision of the W3C XML Plenary and - XML Coordination groups: - - In appreciation for his vision, leadership and dedication - the W3C XML Plenary on this 10th day of February, 2000 - reserves for Jon Bosak in perpetuity the XML name - xml:Father - - + + - This schema defines attributes and an attribute group - suitable for use by - schemas wishing to allow xml:base, xml:lang or xml:space attributes - on elements they define. - - To enable this, such a schema must import this schema - for the XML namespace, e.g. as follows: - <schema . . .> - . . . - <import namespace="http://www.w3.org/XML/1998/namespace" - schemaLocation="http://www.w3.org/2001/03/xml.xsd"/> - - Subsequently, qualified reference to any of the attributes - or the group defined below will have the desired effect, e.g. - - <type . . .> - . . . - <attributeGroup ref="xml:specialAttrs"/> - - will define a type which will schema-validate an instance - element with any of those attributes - + +
+

About the XML namespace

- - In keeping with the XML Schema WG's standard versioning - policy, this schema document will persist at - http://www.w3.org/2004/10/xml.xsd. - At the date of issue it can also be found at - http://www.w3.org/2001/xml.xsd. - The schema document at that URI may however change in the future, - in order to remain compatible with the latest version of XML Schema - itself, or with the XML namespace itself. In other words, if the XML - Schema or XML namespaces change, the version of this document at - http://www.w3.org/2001/xml.xsd will change - accordingly; the version at - http://www.w3.org/2004/10/xml.xsd will not change. +
+

+ This schema document describes the XML namespace, in a form + suitable for import by other schema documents. +

+

+ See + http://www.w3.org/XML/1998/namespace.html and + + http://www.w3.org/TR/REC-xml for information + about this namespace. +

+

+ Note that local names in this namespace are intended to be + defined only by the World Wide Web Consortium or its subgroups. + The names currently defined in this namespace are listed below. + They should not be used with conflicting semantics by any Working + Group, specification, or document instance. +

+

+ See further below in this document for more information about how to refer to this schema document from your own + XSD schema documents and about the + namespace-versioning policy governing this schema document. +

+
+
- + - Attempting to install the relevant ISO 2- and 3-letter - codes as the enumerated possible values is probably never - going to be a realistic possibility. See - RFC 3066 at http://www.ietf.org/rfc/rfc3066.txt and the IANA registry - at http://www.iana.org/assignments/lang-tag-apps.htm for - further information. + +
+ +

lang (as an attribute name)

+

+ denotes an attribute whose value + is a language code for the natural language of the content of + any element; its value is inherited. This name is reserved + by virtue of its definition in the XML specification.

+ +
+
+

Notes

+

+ Attempting to install the relevant ISO 2- and 3-letter + codes as the enumerated possible values is probably never + going to be a realistic possibility. +

+

+ See BCP 47 at + http://www.rfc-editor.org/rfc/bcp/bcp47.txt + and the IANA language subtag registry at + + http://www.iana.org/assignments/language-subtag-registry + for further information. +

+

+ The union allows for the 'un-declaration' of xml:lang with + the empty string. +

+
+ + + + + + + + + + + + +
+ +

space (as an attribute name)

+

+ denotes an attribute whose + value is a keyword indicating what whitespace processing + discipline is intended for the content of the element; its + value is inherited. This name is reserved by virtue of its + definition in the XML specification.

+ +
+ + @@ -109,18 +109,48 @@ - - - - See http://www.w3.org/TR/xmlbase/ for - information about this attribute. + + + +
+ +

base (as an attribute name)

+

+ denotes an attribute whose value + provides a URI to be used as the base for interpreting any + relative URIs in the scope of the element on which it + appears; its value is inherited. This name is reserved + by virtue of its definition in the XML Base specification.

+ +

+ See http://www.w3.org/TR/xmlbase/ + for information about this attribute. +

+
+ - See http://www.w3.org/TR/xml-id/ for - information about this attribute. + +
+ +

id (as an attribute name)

+

+ denotes an attribute whose value + should be interpreted as if declared to be of type ID. + This name is reserved by virtue of its definition in the + xml:id specification.

+ +

+ See http://www.w3.org/TR/xml-id/ + for information about this attribute. +

+
+ @@ -128,6 +158,130 @@ + + + +
+ +

Father (in any context at all)

+ +
+

+ denotes Jon Bosak, the chair of + the original XML Working Group. This name is reserved by + the following decision of the W3C XML Plenary and + XML Coordination groups: +

+
+

+ In appreciation for his vision, leadership and + dedication the W3C XML Plenary on this 10th day of + February, 2000, reserves for Jon Bosak in perpetuity + the XML name "xml:Father". +

+
+
+
+ + + + + +
+

About this schema document

+ +
+

+ This schema defines attributes and an attribute group suitable + for use by schemas wishing to allow xml:base, + xml:lang, xml:space or + xml:id attributes on elements they define. +

+

+ To enable this, such a schema must import this schema for + the XML namespace, e.g. as follows: +

+ 
+          <schema . . .>
+           . . .
+           <import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2001/xml.xsd"/>
+
+

+ or +

+ 
+           <import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2009/01/xml.xsd"/>
+
+

+ Subsequently, qualified reference to any of the attributes or the + group defined below will have the desired effect, e.g. +

+ 
+          <type . . .>
+           . . .
+           <attributeGroup ref="xml:specialAttrs"/>
+
+

+ will define a type which will schema-validate an instance element + with any of those attributes. +

+
+
+ + + + + +
+

Versioning policy for this schema document

+
+

+ In keeping with the XML Schema WG's standard versioning + policy, this schema document will persist at + + http://www.w3.org/2009/01/xml.xsd. +

+

+ At the date of issue it can also be found at + + http://www.w3.org/2001/xml.xsd. +

+

+ The schema document at that URI may however change in the future, + in order to remain compatible with the latest version of XML + Schema itself, or with the XML namespace itself. In other words, + if the XML Schema or XML namespaces change, the version of this + document at + http://www.w3.org/2001/xml.xsd + + will change accordingly; the version at + + http://www.w3.org/2009/01/xml.xsd + + will not change. +

+

+ Previous dated (and unchanging) versions of this schema + document are at: +

+ +
+
+ + + + diff --git a/exist-core/src/test/xquery/transform/catalog.xql b/exist-core/src/test/xquery/transform/catalog.xql new file mode 100644 index 00000000000..6b9180e5489 --- /dev/null +++ b/exist-core/src/test/xquery/transform/catalog.xql @@ -0,0 +1,101 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +(:~ + : Catalog resolution for xsl:import/xsl:include in transform:transform() and fn:transform(). + : + : @see https://github.com/eXist-db/exist/issues/350 + : @see https://github.com/eXist-db/exist/issues/5051 + : @see https://github.com/eXist-db/exist/issues/5052 + : @see https://github.com/eXist-db/exist/issues/5682 + :) +module namespace tc="http://exist-db.org/xquery/test/transform/catalog"; + +declare namespace test="http://exist-db.org/xquery/xqsuite"; + +(:~ + : A non-routable absolute URI (TEST-NET-3, RFC 5737) -- it cannot resolve via the database, an + : xmldb:/EXpath-registered location, or a live network fetch, only via the system catalog entry + : for it in org/exist/validation/catalog.xml. + :) +declare variable $tc:IMPORT_URI := "http://203.0.113.1/transform-catalog-test-lib.xsl"; + +(:~ + : Imports $tc:IMPORT_URI and applies its lib:greet() template against the (otherwise unused) + : context document, shared between both transform functions below so the only difference + : between the two tests is which function consumes the stylesheet. + :) +declare variable $tc:STYLESHEET := + + + + + + ; + +(:~ + : transform:transform() (legacy) must resolve a catalog-redirected xsl:import the same way the + : Xerces/JAXP validation pipeline already does, via XsltURIResolverHelper's resolver chain. + :) +declare + %test:assertEquals("hello") +function tc:legacy-transform-resolves-import-via-catalog() { + transform:transform(bonjourno, $tc:STYLESHEET, ()) +}; + +(:~ + : fn:transform() should resolve a catalog-redirected xsl:import via its compile-time URIResolver. + : + : @see https://github.com/eXist-db/exist/issues/5052 + :) +declare + %test:pending("https://github.com/eXist-db/exist/issues/5052") + %test:assertEquals("hello") +function tc:fn-transform-resolves-import-via-catalog() { + fn:transform(map { + "stylesheet-node": $tc:STYLESHEET, + "source-node": bonjourno + })?output +}; + +(:~ + : fn:transform() should resolve a catalog-redirected document() call made at runtime, via + : SaxonConfiguration's Configuration-level ResourceResolver. + : + : @see https://github.com/eXist-db/exist/issues/5052 + :) +declare + %test:pending("https://github.com/eXist-db/exist/issues/5052") + %test:assertEquals("hello") +function tc:fn-transform-resolves-document-call-via-catalog() { + fn:transform(map { + "stylesheet-node": + + + + + , + "source-node": bonjourno + })?output +}; diff --git a/exist-distribution/pom.xml b/exist-distribution/pom.xml index 8e4ccf29f7f..37b3267843d 100644 --- a/exist-distribution/pom.xml +++ b/exist-distribution/pom.xml @@ -516,12 +516,21 @@ + + + + + + + + + diff --git a/exist-distribution/src/main/config/collection.xconf.init b/exist-distribution/src/main/config/collection.xconf.init index 8933b884e33..266c6c0da4c 100644 --- a/exist-distribution/src/main/config/collection.xconf.init +++ b/exist-distribution/src/main/config/collection.xconf.init @@ -1,5 +1,5 @@ - + - + - + - + + + + + + diff --git a/exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml b/exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml index cd91242753d..8be7eb767ae 100644 --- a/exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml +++ b/exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml @@ -6,7 +6,7 @@ ++ The order of elements within this configuration file is significant. --> + xsi:schemaLocation="http://exist.sourceforge.net/NS/exist file:../../../schema/controller-config.xsd" schemaVersion="1.1.1"> diff --git a/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/XMLSchema.dtd b/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/XMLSchema.dtd new file mode 100644 index 00000000000..64aa2d97019 --- /dev/null +++ b/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/XMLSchema.dtd @@ -0,0 +1,513 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +%xs-datatypes; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/XMLSchema.xsd b/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/XMLSchema.xsd index 575975b412e..21c707cd4a4 100644 --- a/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/XMLSchema.xsd +++ b/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/XMLSchema.xsd @@ -1,2163 +1,1558 @@ - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ]> + + - Part 1 version: Id: structures.xsd,v 1.2 2004/01/15 11:34:25 ht Exp - Part 2 version: Id: datatypes.xsd,v 1.3 2004/01/23 18:11:13 ht Exp + Part 1 version: structures.xsd (rec-20120405) + Part 2 version: datatypes.xsd (rec-20120405) - - + + The schema corresponding to this document is normative, with respect to the syntactic constraints it expresses in the - XML Schema language. The documentation (within <documentation> elements) + XML Schema Definition Language. The documentation (within 'documentation' elements) below, is not normative, but rather highlights important aspects of - the W3C Recommendation of which this is a part - + the W3C Recommendation of which this is a part. - - + See below (at the bottom of this document) for information about + the revision and namespace-versioning policy governing this + schema document. + + + + + The simpleType element and all of its members are defined - towards the end of this schema document + towards the end of this schema document. - - - - + + + Get access to the xml: attribute groups for xml:lang as declared on 'schema' and 'documentation' below - - - - - - + + + + + This type is extended by almost all schema types to allow attributes from other namespaces to be added to user schemas. - - - - - - - - - - - + + + + + + + + + + This type is extended by all types which allow annotation - other than <schema> itself + other than <schema> itself - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + This group is for the elements which occur freely at the top level of schemas. All of their types are based on the "annotated" type by extension. - - - - - - - - - - - - + + + + + + + + + + + This group is for the elements which can self-redefine (see <redefine> below). - - - - - - - - - - - - + + + + + + + + + + + A utility type, not for public use - - - - - - - - - - + + + + + + + + + A utility type, not for public use - - - - - - - - - - + + + + + + + + + A utility type, not for public use - + #all or (possibly empty) subset of {extension, restriction} - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + A utility type, not for public use - - - - - - - - - + + + + + + + + - - + + A utility type, not for public use - + #all or (possibly empty) subset of {extension, restriction, list, union} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - for maxOccurs - - - - - - - - - - - - for all particles - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + for maxOccurs + + + + + + + + + + + + + for all particles + + + + + + + for element, group and attributeGroup, - which both define and reference - - - - - - - - 'complexType' uses this - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + which both define and reference + + + + + + + + 'complexType' uses this + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + - - + + This branch is short for <complexContent> <restriction base="xs:anyType"> ... </restriction> </complexContent> - - - + + + + + - - - - - - - - - - - Will be restricted to required or forbidden - - - - - + + + + + + + + + + Will be restricted to required or prohibited + + + + + Not allowed if simpleContent child is chosen. - May be overriden by setting on complexContent child. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - This choice is added simply to - make this a valid restriction per the REC - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Overrides any setting on complexType parent. - - - - + May be overridden by setting on complexContent child. + + + + + + + + - - - - - - - - - - This choice is added simply to - make this a valid restriction per the REC - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This choice is added simply to + make this a valid restriction per the REC + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Overrides any setting on complexType parent. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This choice is added simply to + make this a valid restriction per the REC + + + + + + + + + + + + + + + + No typeDefParticle group reference - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - + + A utility type, not for public use - + #all or (possibly empty) subset of {substitution, extension, restriction} - - - - - - - - - + + + + + + + - - - - - + + + + + + + + + - - - + - - - - + + + The element element can be used either at the top level to define an element-type binding globally, or within a content model to either reference a globally-defined element or type or declare an element-type binding locally. The ref form is not allowed at the top level. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This type is used for 'alternative' elements. + + + + + + + + + + + + + + + + + group type for explicit groups, named top-level groups and group references - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - group type for the three kinds of group - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - This choice with min/max is here to - avoid a pblm with the Elt:All/Choice/Seq - Particle derivation constraint - - - - - - - - - - restricted max/min - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - Only elements allowed inside - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + - - - - - simple type for the value of the 'namespace' attr of - 'any' and 'anyAttribute' - - - - Value is - ##any - - any non-conflicting WFXML/attribute at all - - ##other - - any non-conflicting WFXML/attribute from - namespace other than targetNS - - ##local - - any unqualified non-conflicting WFXML/attribute - - one or - - any non-conflicting WFXML/attribute from - more URI the listed namespaces - references - (space separated) - - ##targetNamespace or ##local may appear in the above list, to - refer to the targetNamespace of the enclosing - schema or an absent targetNamespace respectively - - - - - - A utility type, not for public use - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + group type for the three kinds of group + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - A subset of XPath expressions for use -in selectors - A utility type, not for public -use - - - - The following pattern is intended to allow XPath - expressions per the following EBNF: - Selector ::= Path ( '|' Path )* - Path ::= ('.//')? Step ( '/' Step )* - Step ::= '.' | NameTest - NameTest ::= QName | '*' | NCName ':' '*' - child:: is also allowed - - - - - - - - - - - - - - - - - - - - - - - A subset of XPath expressions for use -in fields - A utility type, not for public -use - - + + + + - The following pattern is intended to allow XPath - expressions per the same EBNF as for selector, - with the following change: - Path ::= ('.//')? ( Step '/' )* ( Step | '@' NameTest ) - + This choice with min/max is here to + avoid a pblm with the Elt:All/Choice/Seq + Particle derivation constraint - - - - - - - - - - - - - - - - - - - - - - - - - The three kinds of identity constraints, all with - type of or derived from 'keybase'. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - A utility type, not for public use - - A public identifier, per ISO 8879 - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + Only elements allowed inside + - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - notations for use within XML Schema schemas - - - - - - - - - Not the real urType, but as close an approximation as we can - get in the XML representation - - - - - - - - - - First the built-in primitive datatypes. These definitions are for - information only, the real built-in definitions are magic. - - - - For each built-in datatype in this schema (both primitive and - derived) can be uniquely addressed via a URI constructed - as follows: - 1) the base URI is the URI of the XML Schema namespace - 2) the fragment identifier is the name of the datatype - - For example, to address the int datatype, the URI is: - - http://www.w3.org/2001/XMLSchema#int - - Additionally, each facet definition element can be uniquely - addressed via a URI constructed as follows: - 1) the base URI is the URI of the XML Schema namespace - 2) the fragment identifier is the name of the facet - - For example, to address the maxInclusive facet, the URI is: - - http://www.w3.org/2001/XMLSchema#maxInclusive - - Additionally, each facet usage in a built-in datatype definition - can be uniquely addressed via a URI constructed as follows: - 1) the base URI is the URI of the XML Schema namespace - 2) the fragment identifier is the name of the datatype, followed - by a period (".") followed by the name of the facet - - For example, to address the usage of the maxInclusive facet in - the definition of int, the URI is: - - http://www.w3.org/2001/XMLSchema#int.maxInclusive - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - + + + + source="../structures/structures.html#element-choice"/> - - - - - - - - - - - - - - - - - - - - - - + + + + source="../structures/structures.html#element-sequence"/> - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + simple type for the value of the 'namespace' attr of + 'any' and 'anyAttribute' + + + + Value is + ##any - - any non-conflicting WFXML/attribute at all - - - - - - - - - - - - - - - - - - - - - + ##other - - any non-conflicting WFXML/attribute from + namespace other than targetNS - - - - - - - - - - - - - - - - - - - - - + ##local - - any unqualified non-conflicting WFXML/attribute - - - - - - - - - - - - - - - - - - - - - + one or - - any non-conflicting WFXML/attribute from + more URI the listed namespaces + references + (space separated) - - - - - - - - - - - - - - - - + ##targetNamespace or ##local may appear in the above list, to + refer to the targetNamespace of the enclosing + schema or an absent targetNamespace respectively + + + + + A utility type, not for public use - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + A utility type, not for public use - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + A utility type, not for public use - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + A utility type, not for public use + - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + - NOTATION cannot be used directly in a schema; rather a type - must be derived from it by specifying at least one enumeration - facet whose value is the name of a NOTATION declared in the - schema. + A utility type, not for public use - - - + + + + + + + + + + + - - - - Now the derived primitive types - - - - + + + + + + + + + + + + + source="../structures/structures.html#element-attribute"/> - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + source="../structures/structures.html#element-attributeGroup"/> - - - - - - + + + source="../structures/structures.html#element-include"/> - - - - - pattern specifies the content of section 2.12 of XML 1.0e2 - and RFC 3066 (Revised version of RFC 1766). - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + source="../structures/structures.html#element-redefine"/> - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + source="../structures/structures.html#element-override"/> - - - - - - - - - + + + + + + + + + + + + + + + source="../structures/structures.html#element-import"/> - - - - - pattern matches production 7 from the XML spec - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + source="../structures/structures.html#element-selector"/> - - - - - - - - - + + + + + + + A subset of XPath expressions for use +in selectors + A utility type, not for public +use + + + + + + + + + + + + + + + + + + + + + A subset of XPath expressions for use +in fields + A utility type, not for public +use + + + + + + + + + + + + + + + + + + + + + + + - + The three kinds of identity constraints, all with + type of or derived from 'keybase'. + - - - - - pattern matches production 5 from the XML spec - - - - - - - + + + + + + + + source="../structures/structures.html#element-unique"/> - - - - - pattern matches production 4 from the Namespaces in XML spec - - - - - - - + + - + - - - - + + + source="../structures/structures.html#element-keyref"/> - - - - + + + + + + + + + + source="../structures/structures.html#element-notation"/> - - - - + + + + + + + + + + + - + + A utility type, not for public use + + A public identifier, per ISO 8879 - - - - + - - + + source="../structures/structures.html#element-appinfo"/> - - - - - - + + + + + + + + + + source="../structures/structures.html#element-documentation"/> - - - - - - + + + + + + + + + + - - - - + source="../structures/structures.html#element-annotation"/> - - - - - - - + + + + + + + + + + + + + + + notations for use within schema documents + + + + - + + Not the real urType, but as close an approximation as we can + get in the XML representation - - - - - + + + + + - - - - - - - - - + + + In keeping with the XML Schema WG's standard versioning policy, + the material in this schema document will persist at the URI + http://www.w3.org/2012/04/XMLSchema.xsd. - - - - - - - - - + At the date of issue it can also be found at the URI + http://www.w3.org/2009/XMLSchema/XMLSchema.xsd. - - - - - - - - + The schema document at that URI may however change in the future, + in order to remain compatible with the latest version of XSD + and its namespace. In other words, if XSD or the XML Schema + namespace change, the version of this document at + http://www.w3.org/2009/XMLSchema/XMLSchema.xsd will change accordingly; + the version at http://www.w3.org/2012/04/XMLSchema.xsd will not change. - - - - - - - - - - - - + Previous dated (and unchanging) versions of this schema document + include: - - - - - - - - + http://www.w3.org/2012/01/XMLSchema.xsd + (XSD 1.1 Proposed Recommendation) - - - - - - - - + http://www.w3.org/2011/07/XMLSchema.xsd + (XSD 1.1 Candidate Recommendation) - - - - - - - - + http://www.w3.org/2009/04/XMLSchema.xsd + (XSD 1.1 Candidate Recommendation) - - - - - - - - + http://www.w3.org/2004/10/XMLSchema.xsd + (XSD 1.0 Recommendation, Second Edition) - - - - A utility type, not for public use + http://www.w3.org/2001/05/XMLSchema.xsd + (XSD 1.0 Recommendation, First Edition) + + + - - - - - - - - - - - - - - - - - - - #all or (possibly empty) subset of {restriction, union, list} - - + + + + + A utility type, not for public use - - - - - + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + #all or (possibly empty) subset of {restriction, extension, union, list} + + + A utility type, not for public use + + + + + + + + + + + + + + + + + + + + + @@ -2173,7 +1568,6 @@ use - @@ -2181,19 +1575,17 @@ use - + Required at the top level - + - @@ -2209,72 +1601,59 @@ use - + - + source="http://www.w3.org/TR/xmlschema11-2/#element-simpleType"/> + + + + + + An abstract element, representing facets in general. + The facets defined by this spec are substitutable for + this element, and implementation-defined facets should + also name this as a substitution-group head. + - - - - - We should use a substitution group for facets, but - that's ruled out because it would allow users to - add their own, which we're not ready for yet. - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - + + + base attribute and simpleType child are mutually exclusive, but one or other is required - - + + - - - - + + + itemType attribute and simpleType child are mutually exclusive, but one or other is required @@ -2283,19 +1662,18 @@ use + minOccurs="0"/> - - - - + + + memberTypes attribute must be non-empty or there must be at least one simpleType child @@ -2304,7 +1682,7 @@ use + minOccurs="0" maxOccurs="unbounded"/> @@ -2315,71 +1693,88 @@ use - - + - - - - - - - - - - - - - - + + + + + + + + + + + + + source="http://www.w3.org/TR/xmlschema11-2/#element-minExclusive"/> - + + source="http://www.w3.org/TR/xmlschema11-2/#element-minInclusive"/> - - + + source="http://www.w3.org/TR/xmlschema11-2/#element-maxExclusive"/> - + + source="http://www.w3.org/TR/xmlschema11-2/#element-maxInclusive"/> - - - - - - + + + + + + + + + + + + + + + + + - + + source="http://www.w3.org/TR/xmlschema11-2/#element-totalDigits"/> @@ -2388,48 +1783,56 @@ use - + - + + source="http://www.w3.org/TR/xmlschema11-2/#element-fractionDigits"/> - + + source="http://www.w3.org/TR/xmlschema11-2/#element-length"/> - + + source="http://www.w3.org/TR/xmlschema11-2/#element-minLength"/> - + + source="http://www.w3.org/TR/xmlschema11-2/#element-maxLength"/> - - + + source="http://www.w3.org/TR/xmlschema11-2/#element-enumeration"/> - - + + source="http://www.w3.org/TR/xmlschema11-2/#element-whiteSpace"/> @@ -2446,16 +1849,16 @@ use - + - - + + source="http://www.w3.org/TR/xmlschema11-2/#element-pattern"/> @@ -2463,11 +1866,85 @@ use - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In keeping with the XML Schema WG's standard versioning policy, + this schema document will persist at the URI + http://www.w3.org/2012/04/datatypes.xsd. + + At the date of issue it can also be found at the URI + http://www.w3.org/2009/XMLSchema/datatypes.xsd. + + The schema document at that URI may however change in the future, + in order to remain compatible with the latest version of XSD + and its namespace. In other words, if XSD or the XML Schema + namespace change, the version of this document at + http://www.w3.org/2009/XMLSchema/datatypes.xsd will change accordingly; + the version at http://www.w3.org/2012/04/datatypes.xsd will not change. + + Previous dated (and unchanging) versions of this schema document + include: + + http://www.w3.org/2012/01/datatypes.xsd + (XSD 1.1 Proposed Recommendation) + + http://www.w3.org/2011/07/datatypes.xsd + (XSD 1.1 Candidate Recommendation) + + http://www.w3.org/2009/04/datatypes.xsd + (XSD 1.1 Candidate Recommendation) + + http://www.w3.org/2004/10/datatypes.xsd + (XSD 1.0 Recommendation, Second Edition) + + http://www.w3.org/2001/05/datatypes.xsd + (XSD 1.0 Recommendation, First Edition) + + + + + + diff --git a/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/datatypes.dtd b/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/datatypes.dtd new file mode 100644 index 00000000000..f9352bae1c4 --- /dev/null +++ b/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/datatypes.dtd @@ -0,0 +1,222 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/xml.xsd b/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/xml.xsd index de11f7e07b2..aea7d0db0a4 100644 --- a/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/xml.xsd +++ b/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/xml.xsd @@ -1,107 +1,107 @@ - - - - See http://www.w3.org/XML/1998/namespace.html and - http://www.w3.org/TR/REC-xml for information about this namespace. - - This schema document describes the XML namespace, in a form - suitable for import by other schema documents. - - Note that local names in this namespace are intended to be defined - only by the World Wide Web Consortium or its subgroups. The - following names are currently defined in this namespace and should - not be used with conflicting semantics by any Working Group, - specification, or document instance: - - base (as an attribute name): denotes an attribute whose value - provides a URI to be used as the base for interpreting any - relative URIs in the scope of the element on which it - appears; its value is inherited. This name is reserved - by virtue of its definition in the XML Base specification. - - id (as an attribute name): denotes an attribute whose value - should be interpreted as if declared to be of type ID. - The xml:id specification is not yet a W3C Recommendation, - but this attribute is included here to facilitate experimentation - with the mechanisms it proposes. Note that it is _not_ included - in the specialAttrs attribute group. - - lang (as an attribute name): denotes an attribute whose value - is a language code for the natural language of the content of - any element; its value is inherited. This name is reserved - by virtue of its definition in the XML specification. - - space (as an attribute name): denotes an attribute whose - value is a keyword indicating what whitespace processing - discipline is intended for the content of the element; its - value is inherited. This name is reserved by virtue of its - definition in the XML specification. - - Father (in any context at all): denotes Jon Bosak, the chair of - the original XML Working Group. This name is reserved by - the following decision of the W3C XML Plenary and - XML Coordination groups: - - In appreciation for his vision, leadership and dedication - the W3C XML Plenary on this 10th day of February, 2000 - reserves for Jon Bosak in perpetuity the XML name - xml:Father - - + + - This schema defines attributes and an attribute group - suitable for use by - schemas wishing to allow xml:base, xml:lang or xml:space attributes - on elements they define. - - To enable this, such a schema must import this schema - for the XML namespace, e.g. as follows: - <schema . . .> - . . . - <import namespace="http://www.w3.org/XML/1998/namespace" - schemaLocation="http://www.w3.org/2001/03/xml.xsd"/> - - Subsequently, qualified reference to any of the attributes - or the group defined below will have the desired effect, e.g. - - <type . . .> - . . . - <attributeGroup ref="xml:specialAttrs"/> - - will define a type which will schema-validate an instance - element with any of those attributes - + +
+

About the XML namespace

- - In keeping with the XML Schema WG's standard versioning - policy, this schema document will persist at - http://www.w3.org/2004/10/xml.xsd. - At the date of issue it can also be found at - http://www.w3.org/2001/xml.xsd. - The schema document at that URI may however change in the future, - in order to remain compatible with the latest version of XML Schema - itself, or with the XML namespace itself. In other words, if the XML - Schema or XML namespaces change, the version of this document at - http://www.w3.org/2001/xml.xsd will change - accordingly; the version at - http://www.w3.org/2004/10/xml.xsd will not change. +
+

+ This schema document describes the XML namespace, in a form + suitable for import by other schema documents. +

+

+ See + http://www.w3.org/XML/1998/namespace.html and + + http://www.w3.org/TR/REC-xml for information + about this namespace. +

+

+ Note that local names in this namespace are intended to be + defined only by the World Wide Web Consortium or its subgroups. + The names currently defined in this namespace are listed below. + They should not be used with conflicting semantics by any Working + Group, specification, or document instance. +

+

+ See further below in this document for more information about how to refer to this schema document from your own + XSD schema documents and about the + namespace-versioning policy governing this schema document. +

+
+
- + - Attempting to install the relevant ISO 2- and 3-letter - codes as the enumerated possible values is probably never - going to be a realistic possibility. See - RFC 3066 at http://www.ietf.org/rfc/rfc3066.txt and the IANA registry - at http://www.iana.org/assignments/lang-tag-apps.htm for - further information. + +
+ +

lang (as an attribute name)

+

+ denotes an attribute whose value + is a language code for the natural language of the content of + any element; its value is inherited. This name is reserved + by virtue of its definition in the XML specification.

+ +
+
+

Notes

+

+ Attempting to install the relevant ISO 2- and 3-letter + codes as the enumerated possible values is probably never + going to be a realistic possibility. +

+

+ See BCP 47 at + http://www.rfc-editor.org/rfc/bcp/bcp47.txt + and the IANA language subtag registry at + + http://www.iana.org/assignments/language-subtag-registry + for further information. +

+

+ The union allows for the 'un-declaration' of xml:lang with + the empty string. +

+
+ + + + + + + + + + + + +
+ +

space (as an attribute name)

+

+ denotes an attribute whose + value is a keyword indicating what whitespace processing + discipline is intended for the content of the element; its + value is inherited. This name is reserved by virtue of its + definition in the XML specification.

+ +
+ + @@ -109,18 +109,48 @@ - - - - See http://www.w3.org/TR/xmlbase/ for - information about this attribute. + + + +
+ +

base (as an attribute name)

+

+ denotes an attribute whose value + provides a URI to be used as the base for interpreting any + relative URIs in the scope of the element on which it + appears; its value is inherited. This name is reserved + by virtue of its definition in the XML Base specification.

+ +

+ See http://www.w3.org/TR/xmlbase/ + for information about this attribute. +

+
+ - See http://www.w3.org/TR/xml-id/ for - information about this attribute. + +
+ +

id (as an attribute name)

+

+ denotes an attribute whose value + should be interpreted as if declared to be of type ID. + This name is reserved by virtue of its definition in the + xml:id specification.

+ +

+ See http://www.w3.org/TR/xml-id/ + for information about this attribute. +

+
+ @@ -128,6 +158,130 @@ + + + +
+ +

Father (in any context at all)

+ +
+

+ denotes Jon Bosak, the chair of + the original XML Working Group. This name is reserved by + the following decision of the W3C XML Plenary and + XML Coordination groups: +

+
+

+ In appreciation for his vision, leadership and + dedication the W3C XML Plenary on this 10th day of + February, 2000, reserves for Jon Bosak in perpetuity + the XML name "xml:Father". +

+
+
+
+ + + + + +
+

About this schema document

+ +
+

+ This schema defines attributes and an attribute group suitable + for use by schemas wishing to allow xml:base, + xml:lang, xml:space or + xml:id attributes on elements they define. +

+

+ To enable this, such a schema must import this schema for + the XML namespace, e.g. as follows: +

+ 
+          <schema . . .>
+           . . .
+           <import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2001/xml.xsd"/>
+
+

+ or +

+ 
+           <import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2009/01/xml.xsd"/>
+
+

+ Subsequently, qualified reference to any of the attributes or the + group defined below will have the desired effect, e.g. +

+ 
+          <type . . .>
+           . . .
+           <attributeGroup ref="xml:specialAttrs"/>
+
+

+ will define a type which will schema-validate an instance element + with any of those attributes. +

+
+
+ + + + + +
+

Versioning policy for this schema document

+
+

+ In keeping with the XML Schema WG's standard versioning + policy, this schema document will persist at + + http://www.w3.org/2009/01/xml.xsd. +

+

+ At the date of issue it can also be found at + + http://www.w3.org/2001/xml.xsd. +

+

+ The schema document at that URI may however change in the future, + in order to remain compatible with the latest version of XML + Schema itself, or with the XML namespace itself. In other words, + if the XML Schema or XML namespaces change, the version of this + document at + http://www.w3.org/2001/xml.xsd + + will change accordingly; the version at + + http://www.w3.org/2009/01/xml.xsd + + will not change. +

+

+ Previous dated (and unchanging) versions of this schema + document are at: +

+ +
+
+ + + + diff --git a/exist-parent/pom.xml b/exist-parent/pom.xml index 570df3da812..4cab338385e 100644 --- a/exist-parent/pom.xml +++ b/exist-parent/pom.xml @@ -137,6 +137,9 @@ 10.5.0 2.1.3 12.5 + 2.12.2 + 1.2.0 + 10k 6.0.23 2.12.0 4.13.2 @@ -229,6 +232,56 @@ ${saxon.version} + + org.exist-db.thirdparty.xerces + xercesImpl + ${xerces.version} + jdk14-xml-schema-1.1 + + + xml-apis + xml-apis + + + + + + org.exist-db.thirdparty.org.eclipse.wst.xml + xpath2 + ${xpath2.version} + + + + edu.princeton.cup + java-cup + ${java-cup.version} + + + + org.xmlresolver + xmlresolver + ${xmlresolver.version} + + + xml-apis + xml-apis + + + + + + org.xmlresolver + xmlresolver + ${xmlresolver.version} + data + + + xml-apis + xml-apis + + + + com.evolvedbinary.j8fu j8fu @@ -912,6 +965,28 @@ Saxon-HE ${saxon.version} + + + org.exist-db.thirdparty.xerces + xercesImpl + ${xerces.version} + jdk14-xml-schema-1.1 + + + org.exist-db.thirdparty.org.eclipse.wst.xml + xpath2 + ${xpath2.version} + + + edu.princeton.cup + java-cup + ${java-cup.version} + + + org.xmlresolver + xmlresolver + ${xmlresolver.version} + diff --git a/pom.xml b/pom.xml index 9566e6abc41..1d0531bdc70 100644 --- a/pom.xml +++ b/pom.xml @@ -67,6 +67,118 @@ true + + + + org.codehaus.mojo + xml-maven-plugin + + + + schema-governance + none + + transform + + + + + ${project.build.directory}/governance + + context.xml + + ${project.basedir}/schema/governance.xsl + ${project.build.directory}/governance + + + .report.xml + + + + + + + + validate-canonical-instances + validate + + validate + + + + strict + + ${project.basedir}/exist-jetty-config/src/main/resources/webapp/WEB-INF/catalog.xml + + + + ${project.basedir}/exist-distribution/src/main/config + + conf.xml + + ${project.basedir}/schema/conf.xsd + http://www.w3.org/XML/XMLSchema/v1.1 + + + ${project.basedir}/exist-distribution/src/main/config + + collection.xconf.init + + ${project.basedir}/schema/collection.xconf.xsd + http://www.w3.org/XML/XMLSchema/v1.1 + + + ${project.basedir}/exist-distribution/src/main/config + + descriptor.xml + + ${project.basedir}/schema/descriptor.xsd + http://www.w3.org/XML/XMLSchema/v1.1 + + + ${project.basedir}/exist-jetty-config/src/main/resources/webapp/WEB-INF + + controller-config.xml + + ${project.basedir}/schema/controller-config.xsd + http://www.w3.org/XML/XMLSchema/v1.1 + + + ${project.basedir}/exist-core/src/main/resources/org/exist/util + + mime-types.xml + + ${project.basedir}/schema/mime-types.xsd + http://www.w3.org/XML/XMLSchema/v1.1 + + + + ${project.basedir}/schema + + **/*.xsd + + ${project.basedir}/exist-jetty-config/src/main/resources/webapp/WEB-INF/entities/XMLSchema.xsd + http://www.w3.org/XML/XMLSchema/v1.1 + + + + + + diff --git a/schema/GeneratedSchemaVersions.xml b/schema/GeneratedSchemaVersions.xml new file mode 100644 index 00000000000..4db47163365 --- /dev/null +++ b/schema/GeneratedSchemaVersions.xml @@ -0,0 +1,8 @@ + + diff --git a/schema/README.md b/schema/README.md new file mode 100644 index 00000000000..8946c03a329 --- /dev/null +++ b/schema/README.md @@ -0,0 +1,67 @@ +# eXist-db native XML schemas + +XSD schemas for eXist-db configuration and descriptor files. + +## Versioning policy + +Each native schema declares an independent semver on ``. Schema version is **not** tied to the eXist-db product release. + +| Change type | Version bump | +|-------------|--------------| +| Breaking change for existing instance documents | **MAJOR** | +| Backward-compatible addition (new optional element/attribute) | **MINOR** | +| Documentation or non-semantics XSD-only change | **PATCH** (optional) | + +CI fails if a schema or canonical template changes without bumping the paired `xs:schema/@version`. + +Canonical templates may declare an optional **`schemaVersion`** attribute on the root element. When present, its value should match the paired `xs:schema/@version` (native schema semver — not the eXist product release, and not expath package `@version`). Legacy documents without the attribute remain valid; runtime code logs a debug message when it is missing and warns when it differs from the version this build expects. + +| Schema `@version` | `schemaVersion` on template | +|-------------------|----------------------------| +| `conf.xsd` | `` | +| `collection.xconf.xsd` | `` | +| `descriptor.xsd` | `` | +| `mime-types.xsd` | `` | +| `controller-config.xsd` | `` | + +All five schemas `xs:include` the `schemaVersionType` simple type from [`schema-version-type.xsd`](schema-version-type.xsd) rather than each declaring their own copy (it has no `targetNamespace`, so it is pulled in as a chameleon component and inherits each includer's namespace). + +[`SchemaVersion.java`](../exist-core/src/main/java/org/exist/util/SchemaVersion.java)'s version constants are generated at build time from `xs:schema/@version` on the paired XSDs (see [`generate-schema-version.xsl`](generate-schema-version.xsl), wired as `exist-core/pom.xml`'s `schema-version-codegen` execution) — never hand-edit them. [`SchemaVersionSyncTest`](../exist-core/src/test/java/org/exist/util/SchemaVersionSyncTest.java) is a tautology now that the constants can't drift by construction, kept as a guard against the codegen wiring itself silently breaking. [`ci-schema-checks.yml`](../.github/workflows/ci-schema-checks.yml) still triggers on edits to `SchemaVersion.java` itself. + +## Validation + +**Templates vs schemas** — root [`pom.xml`](../pom.xml) binds `xml-maven-plugin:validate` at the `validate` phase. This also runs on every full build via [`ci-test.yml`](../.github/workflows/ci-test.yml) (`mvn test` runs `validate` first). + +[`ci-schema-checks.yml`](../.github/workflows/ci-schema-checks.yml) re-runs `mvn validate` on PRs that touch schemas or canonical templates (fast, path-filtered). + +**Schemas vs the W3C meta-schema** — the same `validate-canonical-instances` execution also validates every `schema/**/*.xsd` as an instance document against the W3C XSD 1.1 meta-schema, catching malformed schema authoring (e.g. a misplaced `xs:assert`, an unresolvable `xsi:type` in `xs:appinfo`) before it ships. Resolution is fully offline: `catalogHandling` is set to `strict` and a `catalogs` entry points at [`catalog.xml`](../exist-jetty-config/src/main/resources/webapp/WEB-INF/catalog.xml), so the meta-schema's own `xs:import` of the `xml:` namespace never reaches the network — a live fetch would fail the build rather than silently succeeding. Idea borrowed from [#5541](https://github.com/eXist-db/exist/issues/5541), where the same catalog trick lets a user validate the well-formedness of their own XSD against the meta-schema. + +The meta-schema files are vendored, not generated: + +| File | Source | Fetched | +|------|--------|---------| +| `entities/XMLSchema.xsd` | (XSD 1.1 structures schema — supersedes the older 2001/2004 XSD 1.0 revision previously bundled here, which had no `xs:assert`/`vc:` support and was otherwise unused) | 2026-06-20 | +| `entities/XMLSchema.dtd`, `entities/datatypes.dtd` | , `.../datatypes.dtd` — internal-subset companions `XMLSchema.xsd`'s `DOCTYPE`/parameter entities pull in | 2026-06-20 | +| `entities/xml.xsd` | — the `xml:` namespace schema `XMLSchema.xsd` itself imports | 2026-06-20 | + +Published under the [W3C Document License](https://www.w3.org/Consortium/Legal/2015/doc-license) (permissive, redistribution allowed). Each file is byte-identical to its upstream source — kept that way deliberately so future re-syncs are a clean diff against W3C's copy, with no local patching. Duplicated under `exist-core/src/test/resources/org/exist/validation/entities/` to mirror that module's own test catalog, matching this repo's existing pattern for the other bundled DTDs/XSDs. + +**Version bumps** — a single Saxon XSLT 2.0 transform, [`governance.xsl`](governance.xsl), does the whole check in one pass: it reads the schema/template pairs straight from `pom.xml`'s `validate-canonical-instances` validationSets, reads changed paths and BASE-revision copies of each XSD via [`unparsed-text()`](https://www.w3.org/TR/xpath-functions-30/#func-unparsed-text)/[`document()`](https://www.w3.org/TR/xslt-30/#document)/[`doc-available()`](https://www.w3.org/TR/xpath-functions-30/#func-doc-available), and fails the build directly with `xsl:message terminate="yes"` (which also prints the GitHub Actions `::error::` annotations) when a paired schema/template changed without its `xs:schema/@version` moving. [`.github/scripts/prepare-governance-context.sh`](../.github/scripts/prepare-governance-context.sh) is pure git plumbing — it resolves the diff base, dumps each schema's BASE-revision content to disk, and writes a small `context.xml` — everything else is XSLT, run via `mvn -N xml:transform@schema-governance`. + +That execution is bound to `phase=none` in [`pom.xml`](../pom.xml), so it never runs on an ordinary `mvn install`/`mvn test`/`mvn validate` — only [`ci-schema-checks.yml`](../.github/workflows/ci-schema-checks.yml) invokes it directly, after running the shim script. Saxon-HE (XSLT 2.0/3.0) is already a `xml-maven-plugin` dependency via `exist-parent/pom.xml`'s `pluginManagement` — no extra CI dependency installation needed (no more `xmllint`/`xsltproc`). + +## Canonical templates + +| Schema | Template | +|--------|----------| +| [`conf.xsd`](conf.xsd) | [`exist-distribution/src/main/config/conf.xml`](../exist-distribution/src/main/config/conf.xml) | +| [`collection.xconf.xsd`](collection.xconf.xsd) | [`collection.xconf.init`](../exist-distribution/src/main/config/collection.xconf.init) | +| [`descriptor.xsd`](descriptor.xsd) | [`descriptor.xml`](../exist-distribution/src/main/config/descriptor.xml) | +| [`controller-config.xsd`](controller-config.xsd) | [`controller-config.xml`](../exist-jetty-config/src/main/resources/webapp/WEB-INF/controller-config.xml) | +| [`mime-types.xsd`](mime-types.xsd) | [`mime-types.xml`](../exist-core/src/main/resources/org/exist/util/mime-types.xml) | + +Other schemas (`users.xsd`, `server.xsd`, `expath-pkg.xsd`, …) apply to runtime or package files, not shipped templates. + +## Distribution + +Schemas ship at `$EXIST_HOME/schema/` ([#6189](https://github.com/eXist-db/exist/issues/6189)). diff --git a/schema/collection.xconf.xsd b/schema/collection.xconf.xsd index fcd8eb16ede..f5522b8eb3b 100644 --- a/schema/collection.xconf.xsd +++ b/schema/collection.xconf.xsd @@ -6,13 +6,15 @@ xmlns:dcterms="http://purl.org/dc/terms/" elementFormDefault="qualified" targetNamespace="http://exist-db.org/collection-config/1.0" - version="1.0.0"> + version="1.2.2"> + + Schema for eXist-db Collection Configuration files /db/system/config/db/**/collection.xconf Schema for eXist-db Collection Configuration Files - 2011-10-09T18:47:21.319+01:00 + 2011-10-09T18:47:21.319+01:00 Adam Retter @@ -28,6 +30,7 @@ + @@ -267,7 +270,7 @@ Trigger Configuration - + diff --git a/schema/conf.xsd b/schema/conf.xsd index 5b1c0b363d9..fe7b7e65922 100644 --- a/schema/conf.xsd +++ b/schema/conf.xsd @@ -8,8 +8,10 @@ - + version="2.1.1"> + + + @@ -1413,6 +1415,7 @@ + diff --git a/schema/controller-config.xsd b/schema/controller-config.xsd index 7ed505eee78..8bb533c9cae 100644 --- a/schema/controller-config.xsd +++ b/schema/controller-config.xsd @@ -4,13 +4,15 @@ xmlns:exist="http://exist.sourceforge.net/NS/exist" elementFormDefault="qualified" targetNamespace="http://exist.sourceforge.net/NS/exist" - version="1.0.0"> + version="1.1.1"> + + diff --git a/schema/descriptor.xsd b/schema/descriptor.xsd index 30d9432d5b6..f5e160a87f3 100644 --- a/schema/descriptor.xsd +++ b/schema/descriptor.xsd @@ -6,7 +6,8 @@ + version="1.2.1"> + @@ -35,6 +36,8 @@ + + \ No newline at end of file diff --git a/schema/expath-pkg-extensions/cxan.xsd b/schema/expath-pkg-extensions/cxan.xsd index dc2864d613c..98440752004 100644 --- a/schema/expath-pkg-extensions/cxan.xsd +++ b/schema/expath-pkg-extensions/cxan.xsd @@ -5,13 +5,14 @@ xmlns:cxan="http://cxan.org/ns/package" xmlns:dcterms="http://purl.org/dc/terms/" elementFormDefault="qualified" - targetNamespace="http://cxan.org/ns/package"> + targetNamespace="http://cxan.org/ns/package" + version="1.0.1"> A schema for the EXPath Packaging CXAN concept. Schema for the EXPath Packaging CXAN concept. - 2013-11-03T11:36:19.343+01:00 + 2013-11-03T11:36:19.343+01:00 Adam Retter diff --git a/schema/expath-pkg-extensions/exist.xsd b/schema/expath-pkg-extensions/exist.xsd index b72e130e3d3..8825340f5dd 100644 --- a/schema/expath-pkg-extensions/exist.xsd +++ b/schema/expath-pkg-extensions/exist.xsd @@ -5,13 +5,14 @@ xmlns:eepkg="http://exist-db.org/ns/expath-pkg" xmlns:dcterms="http://purl.org/dc/terms/" elementFormDefault="qualified" - targetNamespace="http://exist-db.org/ns/expath-pkg"> + targetNamespace="http://exist-db.org/ns/expath-pkg" + version="1.0.1"> A schema for eXist-db extensions to EXPath Packaging. eXist-db extensions to EXPath Packaging - 2013-11-03T11:36:19.343+01:00 + 2013-11-03T11:36:19.343+01:00 Adam Retter diff --git a/schema/expath-pkg-extensions/repo.xsd b/schema/expath-pkg-extensions/repo.xsd index 14ee73383a7..fe26db45a3a 100644 --- a/schema/expath-pkg-extensions/repo.xsd +++ b/schema/expath-pkg-extensions/repo.xsd @@ -5,13 +5,14 @@ xmlns:repo="http://exist-db.org/xquery/repo" xmlns:dcterms="http://purl.org/dc/terms/" elementFormDefault="qualified" - targetNamespace="http://exist-db.org/xquery/repo"> + targetNamespace="http://exist-db.org/xquery/repo" + version="1.0.1"> A schema for eXist-db Package Repository extensions to EXPath Packaging. eXist-db Package Repository extensions to EXPath Packaging - 2013-11-03T11:36:19.343+01:00 + 2013-11-03T11:36:19.343+01:00 Adam Retter diff --git a/schema/expath-pkg.xsd b/schema/expath-pkg.xsd index c62f50cc315..4b338930a4b 100644 --- a/schema/expath-pkg.xsd +++ b/schema/expath-pkg.xsd @@ -5,13 +5,14 @@ xmlns:pkg="http://expath.org/ns/pkg" xmlns:dcterms="http://purl.org/dc/terms/" elementFormDefault="qualified" - targetNamespace="http://expath.org/ns/pkg"> + targetNamespace="http://expath.org/ns/pkg" + version="1.1.1"> A schema for EXPath Packaging (i.e. expath-pkg.xml) file as per the EXPath Packaging System - Candidate Module 9 May 2012' specification. Schema for EXPath Packaging - 2013-11-03T11:36:19.343+01:00 + 2013-11-03T11:36:19.343+01:00 Adam Retter @@ -41,7 +42,7 @@ The name of the package. A package is named using an IRI, as defined by [RFC 3987], excepted any IRI using the file: scheme (most frequent choices are http: and urn: scheme URIs). Note that the definition of IRI excludes relative references. - + diff --git a/schema/generate-schema-version.xsl b/schema/generate-schema-version.xsl new file mode 100644 index 00000000000..878628d51fb --- /dev/null +++ b/schema/generate-schema-version.xsl @@ -0,0 +1,71 @@ + + + + + + + + + + + + + + + + + /* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * GENERATED by schema/generate-schema-version.xsl at build time -- do not edit by hand. + * Source of truth: each canonical XSD's xs:schema/@version, under schema/. + */ +package org.exist.util; + +/** + * Build-time-generated {@code xs:schema/@version} values for the canonical native XSDs -- see + * {@link SchemaVersion}, which exposes these under its own stable, hand-written constant names. + */ +final class GeneratedSchemaVersions { + + private GeneratedSchemaVersions() { + } + + + + + + generate-schema-version.xsl: + has no xs:schema/@version (resolved as ) -- cannot generate . + + static final String + + = " + + "; + + + } + + + + diff --git a/schema/governance.xsl b/schema/governance.xsl new file mode 100644 index 00000000000..f1648fbb6f8 --- /dev/null +++ b/schema/governance.xsl @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + governance.xsl: no xml-maven-plugin execution with + id='validate-canonical-instances' found in — the execution id + has drifted, or $pom-uri is wrong. Schema governance cannot run blind. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Schema governance failed ( error(s) above). + + + + diff --git a/schema/mime-types.xsd b/schema/mime-types.xsd index 7c1b78b18eb..be05282158b 100644 --- a/schema/mime-types.xsd +++ b/schema/mime-types.xsd @@ -6,7 +6,8 @@ + version="1.2.1"> + @@ -17,7 +18,7 @@ - + @@ -55,6 +56,7 @@ + \ No newline at end of file diff --git a/schema/schema-version-type.xsd b/schema/schema-version-type.xsd new file mode 100644 index 00000000000..ef64c96a97b --- /dev/null +++ b/schema/schema-version-type.xsd @@ -0,0 +1,22 @@ + + + + + + + + + Native XSD semver; mirrors the paired schema's xs:schema/@version (not eXist product version). See schema/README.md. + + + + + + +