Generate test conf.xml/controller-config.xml fixtures from canonical at build time

exist-ant/pom.xml

@@ -108,6 +108,13 @@
             <testResource>
                 <directory>src/test/resources-filtered</directory>
                 <filtering>true</filtering>
+                <excludes>
+                    <exclude>*.xsl</exclude>
+                </excludes>
+            </testResource>
+            <testResource>
+                <directory>${project.build.directory}/generated-test-resources</directory>
+                <filtering>true</filtering>
             </testResource>
         </testResources>
     </build>

exist-ant/src/test/resources-filtered/conf-fixture.xsl

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    eXist-db Open Source Native XML Database
+    Copyright (C) 2001 The eXist-db Authors
+
+    info@exist-db.org
+    http://www.exist-db.org
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+-->
+<!-- Generates this module's test conf.xml from canonical, see schema/generate-conf-fixture.xsl. -->
+<xsl:stylesheet version="3.0"
+    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+    xmlns:xs="http://www.w3.org/2001/XMLSchema"
+    exclude-result-prefixes="xsl xs">
+
+    <xsl:import href="urn:exist-db:codegen:generate-conf-fixture"/>
+
+    <xsl:param name="keep-modules" as="xs:string*" select="(
+        'http://exist-db.org/xquery/inspection',
+        'http://exist-db.org/xquery/response',
+        'http://exist-db.org/xquery/system',
+        'http://exist-db.org/xquery/util',
+        'http://exist-db.org/xquery/xmldb',
+        'http://www.w3.org/2005/xpath-functions/map'
+    )"/>
+
+</xsl:stylesheet>

exist-core/pom.xml

@@ -671,10 +671,22 @@
             <testResource>
                 <directory>src/test/resources</directory>
                 <filtering>false</filtering>
+                <excludes>
+                    <!-- Only the codegen stylesheets, not unrelated .xsl test fixtures
+                         like tei-toc.xsl/transform-catalog-test-lib.xsl elsewhere in this tree. -->
+                    <exclude>**/*-fixture.xsl</exclude>
+                </excludes>
             </testResource>
             <testResource>
                 <directory>src/test/resources-filtered</directory>
                 <filtering>true</filtering>
+                <excludes>
+                    <exclude>**/*.xsl</exclude>
+                </excludes>
+            </testResource>
+            <testResource>
+                <directory>${project.build.directory}/generated-test-resources</directory>
+                <filtering>true</filtering>
             </testResource>
         </testResources>
 
@@ -1082,6 +1094,150 @@ The BaseX Team. The original license statement is also included below.]]></pream
                 </executions>
             </plugin>
 
+            <!--
+                Generates org/exist/util/GeneratedSchemaVersions.java from each canonical XSD's
+                xs:schema/@version (see ../schema/generate-schema-version.xsl), runs on every
+                normal build. Unlike ../schema/governance.xsl (phase=none, CI-invoked-only), this
+                means SchemaVersion.java's version constants can never silently drift from
+                schema/*.xsd.
+            -->
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>xml-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>schema-version-codegen</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>transform</goal>
+                        </goals>
+                        <configuration>
+                            <forceCreation>true</forceCreation>
+                            <transformationSets>
+                                <transformationSet>
+                                    <dir>${project.basedir}/../schema</dir>
+                                    <includes>
+                                        <include>GeneratedSchemaVersions.xml</include>
+                                    </includes>
+                                    <stylesheet>${project.basedir}/../schema/generate-schema-version.xsl</stylesheet>
+                                    <outputDir>${project.build.directory}/generated-sources/schema-version/org/exist/util</outputDir>
+                                    <fileMappers>
+                                        <fileMapper implementation="org.codehaus.plexus.components.io.filemappers.FileExtensionMapper">
+                                            <targetExtension>.java</targetExtension>
+                                        </fileMapper>
+                                    </fileMappers>
+                                </transformationSet>
+                            </transformationSets>
+                        </configuration>
+                    </execution>
+                    <!--
+                        Generates this module's test controller-config.xml from the
+                        exist-jetty-config standalone-webapp template via
+                        src/test/resources/standalone-webapp/WEB-INF/controller-config-fixture.xsl
+                        (which imports schema/generate-controller-config-fixture.xsl, see that
+                        stylesheet's header). Runs before process-test-resources.
+                    -->
+                    <execution>
+                        <id>controller-config-fixture-codegen</id>
+                        <phase>generate-test-resources</phase>
+                        <goals>
+                            <goal>transform</goal>
+                        </goals>
+                        <configuration>
+                            <forceCreation>true</forceCreation>
+                            <transformationSets>
+                                <transformationSet>
+                                    <dir>${project.basedir}/../exist-jetty-config/src/main/resources/standalone-webapp/WEB-INF</dir>
+                                    <includes>
+                                        <include>controller-config.xml</include>
+                                    </includes>
+                                    <stylesheet>${project.basedir}/src/test/resources/standalone-webapp/WEB-INF/controller-config-fixture.xsl</stylesheet>
+                                    <outputDir>${project.build.directory}/generated-test-resources/standalone-webapp/WEB-INF</outputDir>
+                                </transformationSet>
+                            </transformationSets>
+                        </configuration>
+                    </execution>
+                    <!--
+                        Generates this module's test conf.xml/collection.xconf.init fixtures from
+                        canonical via the matching src/test/resources-filtered/.../conf-fixture.xsl
+                        (which imports ../schema/generate-conf-fixture.xsl, see that stylesheet's
+                        header). Runs before process-test-resources, so the standard testResource
+                        filtering above (the ${basedir} substitution) still applies.
+                    -->
+                    <execution>
+                        <id>conf-fixture-codegen</id>
+                        <phase>generate-test-resources</phase>
+                        <goals>
+                            <goal>transform</goal>
+                        </goals>
+                        <configuration>
+                            <forceCreation>true</forceCreation>
+                            <transformationSets>
+                                <transformationSet>
+                                    <dir>${project.basedir}/../exist-distribution/src/main/config</dir>
+                                    <includes>
+                                        <include>conf.xml</include>
+                                    </includes>
+                                    <stylesheet>${project.basedir}/src/test/resources-filtered/conf-fixture.xsl</stylesheet>
+                                    <outputDir>${project.build.directory}/generated-test-resources</outputDir>
+                                </transformationSet>
+                                <transformationSet>
+                                    <dir>${project.basedir}/../exist-distribution/src/main/config</dir>
+                                    <includes>
+                                        <include>conf.xml</include>
+                                    </includes>
+                                    <stylesheet>${project.basedir}/src/test/resources-filtered/org/exist/xquery/conf-fixture.xsl</stylesheet>
+                                    <outputDir>${project.build.directory}/generated-test-resources/org/exist/xquery</outputDir>
+                                </transformationSet>
+                                <transformationSet>
+                                    <dir>${project.basedir}/../exist-distribution/src/main/config</dir>
+                                    <includes>
+                                        <include>conf.xml</include>
+                                    </includes>
+                                    <stylesheet>${project.basedir}/src/test/resources-filtered/org/exist/storage/statistics/conf-fixture.xsl</stylesheet>
+                                    <outputDir>${project.build.directory}/generated-test-resources/org/exist/storage/statistics</outputDir>
+                                </transformationSet>
+                                <transformationSet>
+                                    <dir>${project.basedir}/../exist-distribution/src/main/config</dir>
+                                    <includes>
+                                        <include>conf.xml</include>
+                                    </includes>
+                                    <stylesheet>${project.basedir}/src/test/resources-filtered/org/exist/xquery/functions/transform/conf-fixture.xsl</stylesheet>
+                                    <outputDir>${project.build.directory}/generated-test-resources/org/exist/xquery/functions/transform</outputDir>
+                                </transformationSet>
+                                <transformationSet>
+                                    <dir>${project.basedir}/../exist-distribution/src/main/config</dir>
+                                    <includes>
+                                        <include>collection.xconf.init</include>
+                                    </includes>
+                                    <stylesheet>${project.basedir}/../schema/generate-conf-fixture.xsl</stylesheet>
+                                    <outputDir>${project.build.directory}/generated-test-resources</outputDir>
+                                </transformationSet>
+                            </transformationSets>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <version>3.6.0</version>
+                <executions>
+                    <execution>
+                        <id>add-schema-version-source</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${project.build.directory}/generated-sources/schema-version</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>

exist-core/src/main/java/org/exist/Namespaces.java

@@ -39,6 +39,7 @@ public interface Namespaces {
 	String SCHEMA_NS = XMLConstants.W3C_XML_SCHEMA_NS_URI;
     String SCHEMA_DATATYPES_NS = "http://www.w3.org/2001/XMLSchema-datatypes";
 	String SCHEMA_INSTANCE_NS = XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI;
+    String XSD_1_1_NS = "http://www.w3.org/XML/XMLSchema/v1.1";
 
     // Move this here from Function.BUILTIN_FUNCTION_NS? /ljo
 	String XPATH_FUNCTIONS_NS = "http://www.w3.org/2005/xpath-functions";	

exist-core/src/main/java/org/exist/collections/IndexInfo.java

@@ -96,13 +96,30 @@ void setReader(final XMLReader reader, final EntityResolver entityResolver) thro
         if(entityResolver != null) {
             reader.setEntityResolver(entityResolver);
         }
-        final LexicalHandler lexicalHandler = docTriggers == null ? indexer : docTriggers;
-        final ContentHandler contentHandler = docTriggers == null ? indexer : docTriggers;
-        reader.setProperty(Namespaces.SAX_LEXICAL_HANDLER, lexicalHandler);
-        reader.setContentHandler(contentHandler);
+        reader.setProperty(Namespaces.SAX_LEXICAL_HANDLER, getLexicalHandler());
+        reader.setContentHandler(getContentHandler());
         reader.setErrorHandler(indexer);
     }
 
+    /**
+     * The same content handler {@link #setReader(XMLReader, EntityResolver)} wires onto an
+     * {@link XMLReader} -- exposed so callers that validate via a {@link javax.xml.validation.ValidatorHandler}
+     * instead of an {@code XMLReader} (which has no equivalent {@code setReader}-style helper) can feed
+     * the same indexing/trigger pipeline through a SAX-driven validation pass.
+     */
+    ContentHandler getContentHandler() {
+        return docTriggers == null ? indexer : docTriggers;
+    }
+
+    /**
+     * The same lexical handler {@link #setReader(XMLReader, EntityResolver)} wires onto an
+     * {@link XMLReader} -- see {@link #getContentHandler()} for why this is also exposed
+     * separately.
+     */
+    LexicalHandler getLexicalHandler() {
+        return docTriggers == null ? indexer : docTriggers;
+    }
+
     void setDOMStreamer(final DOMStreamer streamer) {
         this.streamer = streamer;
         if (docTriggers == null) {

exist-core/src/main/java/org/exist/collections/MutableCollection.java

@@ -117,6 +117,15 @@ public class MutableCollection implements Collection {
     private final Permission permissions;
     @Deprecated private CollectionMetadata collectionMetadata = null;
 
+    /**
+     * Discards all cached XSD 1.1 schema-by-namespace resolutions -- called alongside
+     * {@code Jaxp.clearXsd11DetectionCache()} by {@code validation:clear-grammar-cache()}
+     * so one admin action clears both XSD-1.1-detection caches, not just the schemaLocation-hint one.
+     */
+    public static void clearXsd11SchemaByNamespaceCache() {
+        Xsd11ValidationHelper.clearSchemaCache();
+    }
+
     /**
      * Constructs a Collection Object (not yet persisted)
      *
@@ -1125,9 +1134,8 @@ public void storeDocument(final Txn transaction, final DBBroker broker, final Xm
             // Store XML Document
 
             final BiConsumer2E<XMLReader, IndexInfo, SAXException, EXistException> validatorFn = (xmlReader1, validateIndexInfo) -> {
-                validateIndexInfo.setReader(xmlReader1, null);
                 try {
-                      xmlReader1.parse(source);
+                    Xsd11ValidationHelper.parseOrValidateXmlSource(broker, xmlReader1, validateIndexInfo, source);
                 } catch(final SAXException e) {
                     throw new SAXException("The XML parser reported a problem: " + e.getMessage(), e);
                 } catch(final IOException e) {
@@ -1137,8 +1145,7 @@ public void storeDocument(final Txn transaction, final DBBroker broker, final Xm
 
             final BiConsumer2E<XMLReader, IndexInfo, SAXException, EXistException> parserFn = (xmlReader1, storeIndexInfo) -> {
                 try {
-                    storeIndexInfo.setReader(xmlReader1, null);
-                    xmlReader1.parse(source);
+                    Xsd11ValidationHelper.parseOrValidateXmlSource(broker, xmlReader1, storeIndexInfo, source);
                 } catch(final IOException e) {
                     throw new EXistException(e);
                 }

exist-core/src/main/java/org/exist/collections/Xsd11SchemaCache.java

@@ -0,0 +1,66 @@
+/*
+ * eXist-db Open Source Native XML Database
+ * Copyright (C) 2001 The eXist-db Authors
+ *
+ * info@exist-db.org
+ * http://www.exist-db.org
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+package org.exist.collections;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+
+import javax.annotation.Nullable;
+import javax.xml.validation.Schema;
+import java.util.Optional;
+
+/**
+ * Per-namespace cache of whether the system catalog's grammar for that namespace needs an
+ * XSD 1.1-capable loader -- see {@link Xsd11ValidationHelper#resolveXsd11SchemaForNamespace}.
+ * A cache miss ({@link #get(String)} returning {@code null}) means "not yet resolved"; a hit of
+ * {@code Optional.empty()} means "resolved: the standard XSD 1.0 pipeline handles this namespace
+ * fine".
+ * <p>
+ * Namespaces reaching this cache are catalog-registered (a finite, admin-controlled set, never
+ * attacker-influenced), so unlike {@link org.exist.validation.Xsd11SchemaDetection}'s
+ * location-driven cache, no eviction/bounding is needed here -- entries live for the life of the
+ * JVM, cleared only via {@link #clear()}.
+ */
+final class Xsd11SchemaCache {
+
+    private static final Cache<String, Optional<Schema>> CACHE = Caffeine.newBuilder().build();
+
+    private Xsd11SchemaCache() {
+    }
+
+    /**
+     * @return the cached resolution for {@code namespace}, or {@code null} if nothing has been
+     * cached for it yet.
+     */
+    @Nullable
+    static Optional<Schema> get(final String namespace) {
+        return CACHE.getIfPresent(namespace);
+    }
+
+    static void put(final String namespace, final Optional<Schema> schema) {
+        CACHE.put(namespace, schema);
+    }
+
+    static void clear() {
+        CACHE.invalidateAll();
+    }
+}