diff --git a/build-logic/plugins/src/main/groovy/org/apache/grails/buildsrc/SbomPlugin.groovy b/build-logic/plugins/src/main/groovy/org/apache/grails/buildsrc/SbomPlugin.groovy index 1b487934e86..4209e8d3fbf 100644 --- a/build-logic/plugins/src/main/groovy/org/apache/grails/buildsrc/SbomPlugin.groovy +++ b/build-logic/plugins/src/main/groovy/org/apache/grails/buildsrc/SbomPlugin.groovy @@ -107,6 +107,12 @@ class SbomPlugin implements Plugin { 'pkg:maven/jline/jline@2.14.6?type=jar' : 'BSD-2-Clause', // maps incorrectly because of https://github.com/CycloneDX/cyclonedx-core-java/issues/205 'pkg:maven/opensymphony/sitemesh@2.6.0?type=jar' : 'OpenSymphony', // custom license approved by legal LEGAL-707 'pkg:maven/org.antlr/antlr4-runtime@4.7.2?type=jar' : 'BSD-3-Clause', // maps incorrectly because of https://github.com/CycloneDX/cyclonedx-core-java/issues/205 + // mongo-java-server declares only "The BSD License" so it maps to BSD-4-Clause for the same + // reason as the org.jline group below. https://github.com/bwaldvogel/mongo-java-server/blob/main/LICENSE + // has three numbered clauses and no advertising clause, making it BSD-3-Clause. + 'pkg:maven/de.bwaldvogel/mongo-java-server@1.47.0?type=jar' : 'BSD-3-Clause', + 'pkg:maven/de.bwaldvogel/mongo-java-server-core@1.47.0?type=jar' : 'BSD-3-Clause', + 'pkg:maven/de.bwaldvogel/mongo-java-server-memory-backend@1.47.0?type=jar': 'BSD-3-Clause', // The whole org.jline group declares "The BSD License", which maps incorrectly because of // https://github.com/CycloneDX/cyclonedx-core-java/issues/205 - the POMs point at BSD-3-Clause. // jline.version tracks the JLine version Groovy ships, so every module resolves to one version. diff --git a/gradle.properties b/gradle.properties index e201edf6563..b49b31f28fd 100644 --- a/gradle.properties +++ b/gradle.properties @@ -26,6 +26,8 @@ githubSlug=apache/grails-core apacheDsVersion=1.5.4 apacheMavenVersion=3.9.9 apacheMavenResolverVersion=1.9.22 +flapdoodleVersion=4.33.0 +mongoJavaServerVersion=1.47.0 commonsValidatorVersion=1.9.0 concurrentlinkedhashmapLruVersion=1.4.2 defaultElImplementationVersion=5.0.0 diff --git a/gradle/publish-root-config.gradle b/gradle/publish-root-config.gradle index 6a976480f4f..a8077057fe3 100644 --- a/gradle/publish-root-config.gradle +++ b/gradle/publish-root-config.gradle @@ -151,6 +151,7 @@ def publishedProjects = [ 'grails-data-mongodb', 'grails-data-mongodb-bson', 'grails-data-mongodb-core', + 'grails-data-mongodb-embedded', 'grails-data-mongodb-ext', 'grails-data-mongodb-gson-templates', 'grails-data-mongodb-spring-boot', diff --git a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java index cc9ccf97c71..1be229be4e9 100644 --- a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java +++ b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java @@ -46,6 +46,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; +import org.springframework.context.SmartLifecycle; import org.springframework.context.support.StaticMessageSource; import org.springframework.core.env.PropertyResolver; import org.springframework.transaction.PlatformTransactionManager; @@ -114,7 +115,7 @@ * @author Graeme Rocher * @since 1.0 */ -public class MongoDatastore extends AbstractDatastore implements MappingContext.Listener, Closeable, StatelessDatastore, MultipleConnectionSourceCapableDatastore, MultiTenantCapableDatastore, TransactionCapableDatastore { +public class MongoDatastore extends AbstractDatastore implements MappingContext.Listener, Closeable, StatelessDatastore, MultipleConnectionSourceCapableDatastore, MultiTenantCapableDatastore, TransactionCapableDatastore, SmartLifecycle { public static final String SETTING_DATABASE_NAME = MongoSettings.SETTING_DATABASE_NAME; public static final String SETTING_CONNECTION_STRING = MongoSettings.SETTING_CONNECTION_STRING; @@ -149,7 +150,12 @@ public class MongoDatastore extends AbstractDatastore implements MappingContext. private static final int INDEX_OPTIONS_CONFLICT_CODE = 85; public static final String CODEC_ENGINE = MongoConstants.CODEC_ENGINE; - protected final MongoClient mongo; + /** + * Not final because {@link #start()} replaces it after a CRaC restore. Everything other + * than construction reaches it through {@link #getMongoClient()}, so a replacement is + * picked up without anything else having to be told. + */ + protected volatile MongoClient mongo; protected final String defaultDatabase; protected final Map mongoCollections = new ConcurrentHashMap<>(); protected final Map mongoDatabases = new ConcurrentHashMap<>(); @@ -1158,9 +1164,74 @@ public void persistentEntityAdded(PersistentEntity entity) { initializeIndices(entity); } + /** + * Below the web server's phase so the client outlives the requests using it, and above + * {@code EmbeddedMongoLifecycle.PHASE} so an embedded server outlives this client: + * Spring starts in ascending phase order and stops in descending. + */ + public static final int LIFECYCLE_PHASE = -1000; + + private volatile boolean running = true; + + /** + * Closes the {@link MongoClient} so the process can be checkpointed. + * + *

CRaC refuses to checkpoint a process holding open sockets, and a connected driver + * holds one per pooled connection plus its server monitors. Closing the client shuts the + * monitor threads down and releases every socket, which nothing else in the driver + * offers: draining the pool leaves the monitors connected. + * + *

A client the application supplied is left alone. Its lifecycle belongs to whoever + * created it, and this datastore stays {@link #isRunning() running} so that + * {@link #start()} does not later replace something it does not own. + */ + @Override + public void stop() { + if (!this.running || !ownsClient()) { + return; + } + this.mongo.close(); + this.running = false; + } + + /** + * Builds a replacement {@link MongoClient} after a restore, using the same factory and + * configuration the original was built from, so settings applied at startup still apply. + */ + @Override + public void start() { + if (this.running) { + return; + } + this.mongo = connectionSources.getFactory() + .create(ConnectionSource.DEFAULT, connectionSources.getBaseConfiguration()) + .getSource(); + this.running = true; + } + + @Override + public boolean isRunning() { + return this.running; + } + + @Override + public int getPhase() { + return LIFECYCLE_PHASE; + } + + /** + * Whether GORM created the client and may close it, as opposed to it having been handed + * in by the application. + */ + private boolean ownsClient() { + ConnectionSource source = connectionSources.getDefaultConnectionSource(); + return !(source instanceof DefaultConnectionSource) || ((DefaultConnectionSource) source).isCloseable(); + } + @Override @PreDestroy public void close() { + MongoClient current = this.mongo; try { super.destroy(); } catch (Exception e) { @@ -1170,6 +1241,11 @@ public void close() { if (connectionSources != null) { connectionSources.close(); } + // connectionSources closes the client it was built with, which is no longer the + // one in use once a restore has replaced it. + if (current != null && ownsClient()) { + current.close(); + } } catch (IOException e) { LOG.error("There was an error shutting down GORM for an entity: " + e.getMessage(), e); } finally { diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/MongoDatastoreLifecycleSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/MongoDatastoreLifecycleSpec.groovy new file mode 100644 index 00000000000..94ec17ba657 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/MongoDatastoreLifecycleSpec.groovy @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.mapping.mongo + +import java.util.concurrent.TimeUnit + +import com.mongodb.MongoClientSettings +import com.mongodb.MongoTimeoutException +import com.mongodb.client.MongoClient + +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.config.MongoMappingContext + +import spock.lang.Specification + +/** + * Covers the {@code SmartLifecycle} contract the datastore takes part in. + * + *

CRaC refuses to checkpoint a process holding open sockets, and a connected driver holds + * one per pooled connection plus its server monitors. Spring stops lifecycle beans before the + * checkpoint and starts them again after the restore, so closing the client on stop is what + * lets an application using MongoDB be snapshotted at all -- and building a replacement on + * start is what leaves the restored process able to query anything. + * + *

No server is needed to tell an open client from a closed one: see {@link #closed}. + */ +class MongoDatastoreLifecycleSpec extends Specification { + + void 'a datastore is running from the moment it is built'() { + given: + MongoDatastore datastore = ownedClientDatastore() + + expect: + datastore.running + + and: 'stopping after the web server and before the embedded MongoDB it may be talking to' + datastore.phase == MongoDatastore.LIFECYCLE_PHASE + datastore.phase < 0 + + cleanup: + datastore.close() + } + + void 'stopping closes the client GORM owns, which is what releases its sockets'() { + given: + MongoDatastore datastore = ownedClientDatastore() + MongoClient client = datastore.mongoClient + + expect: 'the client is usable to begin with' + !closed(client) + + when: 'the checkpoint stops it' + datastore.stop() + + then: 'draining the pool would leave the monitors connected, so the client itself is closed' + !datastore.running + closed(client) + + cleanup: + datastore.close() + } + + void 'starting after a stop builds a replacement from the same configuration'() { + given: + MongoDatastore datastore = ownedClientDatastore() + MongoClient original = datastore.mongoClient + datastore.stop() + + when: 'the restore starts it again' + datastore.start() + + then: + datastore.running + + and: 'a closed client cannot be reopened, so the restored process gets a new one' + !datastore.mongoClient.is(original) + !closed(datastore.mongoClient) + + cleanup: + datastore.close() + } + + void 'closing after a restore closes the replacement rather than only the client it replaced'() { + given: 'a datastore that has been through a checkpoint and a restore' + MongoDatastore datastore = ownedClientDatastore() + datastore.stop() + datastore.start() + MongoClient restored = datastore.mongoClient + + when: 'the application shuts down for real' + datastore.close() + + then: 'the connection sources only know the client they were built with, so the one ' + + 'actually in use has to be closed as well rather than left holding sockets' + closed(restored) + } + + void 'stopping an already stopped datastore leaves it alone'() { + given: + MongoDatastore datastore = ownedClientDatastore() + + when: + datastore.stop() + datastore.stop() + + then: + !datastore.running + + when: 'and a running datastore is started again, which would otherwise leak a client' + datastore.start() + MongoClient restored = datastore.mongoClient + datastore.start() + + then: + datastore.running + datastore.mongoClient.is(restored) + + cleanup: + datastore.close() + } + + void 'a client the application supplied is neither closed nor replaced'() { + given: 'a datastore built around an externally managed MongoClient' + MongoClient supplied = Mock(MongoClient) + MongoDatastore datastore = new MongoDatastore(supplied) + + when: 'the checkpoint stops it' + datastore.stop() + + then: 'whoever created the client owns closing it, checkpoint or not' + 0 * supplied.close() + + and: 'so it stays running, and a later start does not replace something it does not own' + datastore.running + + when: + datastore.start() + + then: + datastore.mongoClient.is(supplied) + + cleanup: + datastore.close() + } + + /** + * Whether the driver has been closed, which needs no MongoDB to answer: selecting a server + * from a closed cluster is rejected outright, while an open client with nothing to connect + * to waits for the server selection timeout and gives up. + */ + private static boolean closed(MongoClient client) { + try { + client.listDatabaseNames().first() + false + } + catch (IllegalStateException ignored) { + true + } + catch (MongoTimeoutException ignored) { + false + } + } + + private static MongoDatastore ownedClientDatastore() { + MongoClientSettings.Builder clientOptions = MongoClientSettings.builder() + .applyToClusterSettings { it.serverSelectionTimeout(50, TimeUnit.MILLISECONDS) } + new MongoDatastore(clientOptions, + DatastoreUtils.createPropertyResolver([:]), + new MongoMappingContext('test')) + } +} diff --git a/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/embeddedMongo.adoc b/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/embeddedMongo.adoc new file mode 100644 index 00000000000..515fcf61ab6 --- /dev/null +++ b/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/embeddedMongo.adoc @@ -0,0 +1,178 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// + +=== Embedded MongoDB + +`grails-data-mongodb-embedded` starts a MongoDB server as the application starts, so +development and testing need neither a MongoDB installation nor Docker. Applications created +by Grails Forge with MongoDB selected include it, asked for by the development and test +environments. + +A server is asked for by the URL, the way an in-memory SQL database is. Where an application +would name a host, it names `embedded` instead: + +[source,groovy] +---- +implementation "org.apache.grails:grails-data-mongodb-embedded" +---- + +[source,yaml] +---- +environments: + development: + grails: + mongodb: + url: mongodb://embedded/bookstore + test: + grails: + mongodb: + url: mongodb://embedded/bookstore + production: + grails: + mongodb: + url: mongodb://localhost:27017/bookstore +---- + +Nothing else switches it on and nothing switches it off. A URL naming a host is served by the +driver and no server is started, so adding the dependency alone never changes how an +application connects — in the same way that an application with `h2` on its classpath and a +PostgreSQL URL never starts H2. + +A port may be given as `mongodb://embedded:27018/bookstore`. Without one, the port is 27017 +offset by however far `server.port` has moved from 8080, so two applications run side by side +without colliding. + +NOTE: A host genuinely named `embedded` cannot be reached this way. Name it by its address or +its fully qualified name instead. + +==== Choosing a backend + +Two backends are supported. + +[cols="1,3,3"] +|=== +| |`in-memory` |`flapdoodle` + +|Included +|Yes, with this module +|No, add `de.flapdoodle.embed:de.flapdoodle.embed.mongo` + +|Startup +|Milliseconds, inside this JVM +|About a second, after downloading a MongoDB binary once to `~/.embedmongo` + +|Fidelity +|Reimplements the wire protocol, so transactions, change streams, `$text` and some `$expr` +operators are unsupported +|A real `mongod`, so everything behaves as it does in production + +|Keeps data +|No +|Yes, with `embedded.mongodb.database-dir` + +|Reported version +|`5.0.0` +|The version started, `8.0` by default +|=== + +`in-memory` is used unless flapdoodle is on the classpath, in which case flapdoodle is +preferred, so adding the dependency is all that is needed to move to a real `mongod`. +Set `embedded.mongodb.backend` to choose explicitly. + +Flapdoodle is not a dependency of this module because it requires `jgrapht`, which is +offered under LGPL-2.1 or EPL-2.0 and so cannot be required by an Apache release. An +application adds it in the same way it chooses a SQL driver: + +[source,groovy] +---- +implementation "de.flapdoodle.embed:de.flapdoodle.embed.mongo:4.33.0" +---- + +==== Configuration + +Whether to start a server, which port, and which database are all read from the URL. These +configure the server behind it. + +[cols="1,2"] +|=== +|Property |Description + +|`embedded.mongodb.backend` +|`in-memory` or `flapdoodle`. Defaults to flapdoodle when it is on the classpath, +otherwise `in-memory`. + +|`embedded.mongodb.database-dir` +|Where to keep the data so it outlives the server. Only the flapdoodle backend supports +this; the `in-memory` backend reports an error rather than discarding the data silently. + +|`embedded.mongodb.version` +|The MongoDB version for backends that can choose one, such as `V8_0` for flapdoodle. + +|`embedded.mongodb.property-names` +|Comma separated properties that may name an embedded server. Defaults to +`grails.mongodb.url`. +|=== + +==== Production + +A generated application names a real host in production, so nothing is started there. + +Where an embedded server in production is genuinely wanted, ask for one in that +environment's url and use flapdoodle with a directory, so the data survives a restart: + +[source,yaml] +---- +environments: + production: + grails: + mongodb: + url: mongodb://embedded/bookstore + embedded: + mongodb: + backend: flapdoodle + database-dir: ./prodMongoDb +---- + +==== Restarts and ports + +A server started by this module runs until the JVM exits, so a Spring Boot DevTools +restart reuses it instead of starting another. Only a server this module started is +reused: if anything else is already holding the port, startup fails with an error naming +the port rather than connecting to an unrelated service. To use a MongoDB started by hand, +name its host in `grails.mongodb.url` as usual. + +==== Outside Grails + +The module is plain Java and knows nothing about Grails beyond the default property name, +so `embedded.mongodb.property-names` makes it usable from any Spring application: + +[source,yaml] +---- +spring: + data: + mongodb: + uri: mongodb://embedded/bookstore +embedded: + mongodb: + property-names: spring.data.mongodb.uri +---- + +Spring Data MongoDB applications are usually better served by flapdoodle's own +`de.flapdoodle.embed.mongo.spring3x` auto configuration, which this module deliberately +does not duplicate. diff --git a/grails-data-mongodb/docs/src/docs/asciidoc/index.adoc b/grails-data-mongodb/docs/src/docs/asciidoc/index.adoc index 6d8f5ebc099..ef9fd17219c 100644 --- a/grails-data-mongodb/docs/src/docs/asciidoc/index.adoc +++ b/grails-data-mongodb/docs/src/docs/asciidoc/index.adoc @@ -48,6 +48,9 @@ include::gettingStarted/withHibernate.adoc[] [[advancedConfig]] include::gettingStarted/advancedConfig.adoc[] +[[embeddedMongo]] +include::gettingStarted/embeddedMongo.adoc[] + [[springBoot]] include::gettingStarted/springBoot.adoc[] diff --git a/grails-data-mongodb/embedded/build.gradle b/grails-data-mongodb/embedded/build.gradle new file mode 100644 index 00000000000..4d7d00f9703 --- /dev/null +++ b/grails-data-mongodb/embedded/build.gradle @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id 'groovy' + id 'java-library' + id 'org.apache.grails.buildsrc.properties' + id 'org.apache.grails.buildsrc.dependency-validator' + id 'org.apache.grails.buildsrc.compile' + id 'org.apache.grails.buildsrc.publish' + id 'org.apache.grails.buildsrc.sbom' + id 'org.apache.grails.buildsrc.vulnerability-scan' + id 'org.apache.grails.gradle.grails-code-style' + id 'org.apache.grails.gradle.grails-jacoco' +} + +version = projectVersion +group = 'org.apache.grails' + +ext { + pomTitle = 'Embedded MongoDB for Grails Data' + pomDescription = 'Starts a real mongod before the application context refreshes and publishes its URL into the configured properties' +} + +dependencies { + + implementation platform(project(':grails-bom')) + + api 'org.springframework:spring-context', { + // api: ApplicationContextInitializer, ConfigurableApplicationContext + } + api "de.bwaldvogel:mongo-java-server:$mongoJavaServerVersion", { // Candidate for grails-bom? + // api: MongoServer, MemoryBackend — the default backend, so it ships with the module + } + + implementation 'org.springframework:spring-core', { + // impl: ConfigurableEnvironment, MapPropertySource, ClassUtils + } + implementation 'org.slf4j:slf4j-api', { + // impl: Logger, LoggerFactory + } + + // This module is written in Java so that a plain Spring Boot application can use it, + // but every published module still produces a groovydoc jar, and groovydoc needs + // Groovy on the compile classpath to infer its own. + compileOnly 'org.apache.groovy:groovy' + + // Flapdoodle runs a real mongod but pulls in jgrapht, offered under LGPL-2.1 or + // EPL-2.0, which an Apache release should not require. Applications that want a real + // mongod add it themselves, the way they pick a SQL driver; FlapdoodleMongoBackend + // keeps every reference to it behind an isAvailable() check. + compileOnly "de.flapdoodle.embed:de.flapdoodle.embed.mongo:$flapdoodleVersion", { + // comp: Mongod, ImmutableMongod, Net, Version, DatabaseDir, MongodArguments + } + + testImplementation "de.flapdoodle.embed:de.flapdoodle.embed.mongo:$flapdoodleVersion" + testImplementation 'org.apache.groovy:groovy' + testImplementation 'org.spockframework:spock-core' + testImplementation 'org.mongodb:mongodb-driver-sync' // proves each backend answers a real driver + + testRuntimeOnly 'org.slf4j:slf4j-nop' +} + +apply { + // Starts a real mongod, so these belong with the other MongoDB suites and skip together. + from rootProject.layout.projectDirectory.file('gradle/mongodb-test-config.gradle') +} diff --git a/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoBackend.java b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoBackend.java new file mode 100644 index 00000000000..f6fcce2a7bd --- /dev/null +++ b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoBackend.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongodb.embedded; + +/** + * A MongoDB server that can be started inside, or alongside, the application process. + * + *

Everything else about running an embedded server is the same whichever one is used: + * choosing a port, reusing a server that is already listening, publishing the URL and + * stopping the server when the JVM exits. Only starting it differs, which is all this + * interface covers. + * + * @author Grails + * @since 8.0 + * @see InMemoryMongoBackend + * @see FlapdoodleMongoBackend + */ +public interface EmbeddedMongoBackend { + + /** + * The name this backend is selected by, through {@code embedded.mongodb.backend}. + * + * @return the backend name + */ + String getName(); + + /** + * Whether the library this backend needs is on the classpath. Only one backend is a + * dependency of this module; the other is added by applications that want it, so this + * is what makes selection possible without either being mandatory. + * + * @return true when this backend can be started + */ + boolean isAvailable(); + + /** + * Starts a server bound to localhost. + * + * @param settings the port to bind and any backend specific options + * @return the running server, for the caller to stop + */ + RunningEmbeddedMongo start(EmbeddedMongoSettings settings); +} diff --git a/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoInitializer.java b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoInitializer.java new file mode 100644 index 00000000000..cc2ec2ee623 --- /dev/null +++ b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoInitializer.java @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongodb.embedded; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.aot.AbstractAotProcessor; +import org.springframework.core.SpringProperties; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.MapPropertySource; +import org.springframework.util.ClassUtils; + +/** + * Starts an embedded MongoDB before the application context refreshes and publishes its + * connection URL into whichever configuration properties the application reads, so an + * application runs without a MongoDB installation and without Docker. + * + *

This is an {@link ApplicationContextInitializer} rather than an auto configuration + * because the URL has to be in the {@code Environment} before the datastore bean that + * reads it is created. + * + *

Asked for by the URL, the way an in-memory SQL database is. An application names the + * embedded server where it would otherwise name a host, and the environment that wants one says + * so where it already says which database to talk to: + * + * 

+ * environments:
+ *     development:
+ *         grails:
+ *             mongodb:
+ *                 url: mongodb://embedded/bookstore
+ *     production:
+ *         grails:
+ *             mongodb:
+ *                 url: mongodb://localhost:27017/bookstore
+ *
+ * + *

Nothing else switches it on, and nothing switches it off: a URL naming a host is served by + * the driver and no server is started, exactly as an application with {@code h2} on the classpath + * and a PostgreSQL URL never starts H2. A port may be given -- {@code mongodb://embedded:27018/db} + * -- and is otherwise chosen below. + * + *

The rest configures the server behind that URL rather than which server to reach: + * + * + * + * + * + * + *
Supported properties
{@code embedded.mongodb.backend}{@code in-memory} or {@code flapdoodle}; + * defaults to flapdoodle when it is on the classpath, otherwise in-memory
{@code embedded.mongodb.property-names}comma separated properties that may + * name an embedded server, {@code grails.mongodb.url} by default
{@code embedded.mongodb.database-dir}keeps the data after the server + * stops, which only the flapdoodle backend can do
{@code embedded.mongodb.version}the MongoDB version, where the backend + * can choose one
+ * + *

A host genuinely named {@code embedded} cannot be reached through these properties; name it + * by its address or its fully qualified name instead. + * + *

Making the target property configurable is what keeps this useful outside Grails + * Data: point {@code property-names} at {@code spring.data.mongodb.uri} and it serves a + * plain Spring Boot application, although Spring Data users are generally better served + * by flapdoodle's own {@code de.flapdoodle.embed.mongo.spring3x} auto configuration, + * which this deliberately does not duplicate. + * + * @author Grails + * @since 8.0 + */ +public class EmbeddedMongoInitializer implements ApplicationContextInitializer { + + /** + * The host that means "start one and connect me to it" rather than an address to reach. + */ + public static final String EMBEDDED_HOST = "embedded"; + + public static final String BACKEND = "embedded.mongodb.backend"; + + public static final String PROPERTY_NAMES = "embedded.mongodb.property-names"; + + public static final String DATABASE_DIR = "embedded.mongodb.database-dir"; + + public static final String VERSION = "embedded.mongodb.version"; + + public static final String DEFAULT_PROPERTY_NAME = "grails.mongodb.url"; + + private static final Logger log = LoggerFactory.getLogger(EmbeddedMongoInitializer.class); + + private static final String PROPERTY_SOURCE_NAME = "embeddedMongoDB"; + + private static final String DEFAULT_DATABASE = "test"; + + private static final int DEFAULT_PORT = 27017; + + private static final int DEFAULT_SERVER_PORT = 8080; + + /** + * A URL asking for an embedded server, with the port and database it asks for. Credentials are + * matched so that one copied from a real connection is still recognised, and then ignored: + * there is nothing to authenticate against. + */ + private static final Pattern EMBEDDED_URL = Pattern.compile( + "^mongodb(?:\\+srv)?://(?:[^@/]*@)?" + EMBEDDED_HOST + "(?::(\\d+))?(?:/([^?]*))?(?:\\?.*)?$"); + + /** + * The servers this JVM started, by port. Keyed rather than a single field because two + * application contexts in one JVM can ask for different ports, and consulted so that a + * devtools restart reuses its own server without mistaking any other listener for one. + */ + private static final Map STARTED = new ConcurrentHashMap<>(); + + /** + * Flapdoodle first, so that adding it to an application is all it takes to move from + * the in-memory reimplementation to a real mongod. + */ + private final List backends; + + public EmbeddedMongoInitializer() { + this(defaultBackends(EmbeddedMongoInitializer.class.getClassLoader())); + } + + /** + * The backends to offer, which is only ever the ones whose library is present. + * + *

Asking a backend whether it is available means holding one, and holding one means loading + * its class -- which resolves the types named in its methods, so constructing the flapdoodle + * backend without flapdoodle fails before it can answer. Since it is deliberately not a + * dependency of this module, that is the ordinary case: the initializer could not be created at + * all, and an application asking for the in-memory server got a NoClassDefFoundError naming a + * library it never asked for.

+ * + *

So the question is asked of the class loader instead, by a name rather than by a type.

+ */ + static List defaultBackends(ClassLoader classLoader) { + List backends = new ArrayList<>(); + if (ClassUtils.isPresent(FlapdoodleMongoBackend.MONGOD_CLASS, classLoader)) { + backends.add(new FlapdoodleMongoBackend()); + } + backends.add(new InMemoryMongoBackend()); + return backends; + } + + EmbeddedMongoInitializer(List backends) { + this.backends = backends; + } + + @Override + public void initialize(ConfigurableApplicationContext applicationContext) { + ConfigurableEnvironment environment = applicationContext.getEnvironment(); + + Set propertyNames = propertyNames(environment); + Matcher asked = firstAskingForEmbedded(environment, propertyNames); + if (asked == null) { + return; + } + if (SpringProperties.getFlag(AbstractAotProcessor.AOT_PROCESSING)) { + // Ahead-of-time processing writes bean definitions out as code. It refreshes a context + // to read them, but nothing in it is meant to run, and a database no one will query is + // of no use to it. Starting one is also unrecoverable: the server listens on a + // non-daemon thread and is stopped only by a JVM shutdown hook, so generation finished + // and then hung, holding the port, until it was killed. + log.debug("Not starting an embedded MongoDB: this is ahead-of-time processing"); + return; + } + + int port = resolvePort(environment, asked.group(1)); + String database = resolveDatabase(asked.group(2)); + + String url; + RunningEmbeddedMongo started = STARTED.get(port); + if (started != null) { + // A devtools restart reuses this JVM and this class is loaded from a jar, so it + // survives in the base classloader along with the server the previous + // application context started. Only a server this initializer started is reused; + // anything else holding the port makes the start below fail with an error that + // says so, rather than publishing a MongoDB url pointing at an unrelated service. + url = "mongodb://" + started.getHost() + ":" + started.getPort() + "/" + database; + log.info("Reusing the embedded MongoDB this JVM already started at {}", url); + } + else { + url = start(environment, port, database); + } + + Map published = new HashMap<>(); + for (String propertyName : propertyNames) { + published.put(propertyName, url); + } + environment.getPropertySources().addFirst(new MapPropertySource(PROPERTY_SOURCE_NAME, published)); + + // Registered as a singleton rather than a bean definition because this runs before + // any definitions are read, and the server it manages already exists by now. The + // JVM shutdown hook above still covers the case where the context never refreshes. + RunningEmbeddedMongo running = STARTED.get(port); + if (running != null) { + applicationContext.getBeanFactory() + .registerSingleton(EmbeddedMongoLifecycle.BEAN_NAME, new EmbeddedMongoLifecycle(running)); + } + } + + /** + * Failures throw rather than returning quietly: falling through would leave the target + * properties pointing at whatever the application configured, which is exactly what + * enabling this was meant to replace. + */ + private String start(ConfigurableEnvironment environment, int port, String database) { + EmbeddedMongoBackend backend = selectBackend(environment); + EmbeddedMongoSettings settings = new EmbeddedMongoSettings(port, + environment.getProperty(VERSION), environment.getProperty(DATABASE_DIR)); + + RunningEmbeddedMongo running; + try { + running = backend.start(settings); + } + catch (IllegalStateException ex) { + throw ex; + } + catch (Exception ex) { + throw new IllegalStateException("Failed to start the " + backend.getName() + + " embedded MongoDB on port " + port + ", which something else may already be using. " + + "Name a free port as mongodb://" + EMBEDDED_HOST + ":/, or name a host " + + "instead of " + EMBEDDED_HOST + " to use an external MongoDB.", ex); + } + + STARTED.put(port, running); + + // Registered on the JVM rather than the application context so that it survives a + // devtools restart and fires only once, when the JVM itself exits. + Runtime.getRuntime().addShutdownHook(new Thread(running::stop)); + + String url = "mongodb://" + running.getHost() + ":" + running.getPort() + "/" + database; + log.info("Embedded MongoDB started at {} using the {} backend", url, backend.getName()); + return url; + } + + private EmbeddedMongoBackend selectBackend(ConfigurableEnvironment environment) { + String requested = environment.getProperty(BACKEND); + if (requested != null && !requested.isEmpty()) { + for (EmbeddedMongoBackend backend : this.backends) { + if (backend.getName().equals(requested)) { + if (!backend.isAvailable()) { + throw new IllegalStateException(BACKEND + "=" + requested + + " but its library is not on the classpath. Add it as a dependency, or choose one of " + + availableNames() + "."); + } + return backend; + } + } + if (FlapdoodleMongoBackend.NAME.equals(requested)) { + throw new IllegalStateException(BACKEND + "=" + requested + + " but its library is not on the classpath. Add it as a dependency, or choose one of " + + availableNames() + "."); + } + throw new IllegalStateException(BACKEND + "=" + requested + " is not a known backend. Use one of " + + this.backends.stream().map(EmbeddedMongoBackend::getName).collect(Collectors.joining(", ")) + + "."); + } + + for (EmbeddedMongoBackend backend : this.backends) { + if (backend.isAvailable()) { + return backend; + } + } + throw new IllegalStateException("A url asked for " + EMBEDDED_HOST + + " but no embedded MongoDB backend is on the classpath. " + + "Add de.bwaldvogel:mongo-java-server for an in-memory server, or " + + "de.flapdoodle.embed:de.flapdoodle.embed.mongo for a real mongod."); + } + + private List availableNames() { + List names = new ArrayList<>(); + for (EmbeddedMongoBackend backend : this.backends) { + if (backend.isAvailable()) { + names.add(backend.getName()); + } + } + return names; + } + + private Set propertyNames(ConfigurableEnvironment environment) { + Set propertyNames = new LinkedHashSet<>(); + for (String propertyName : environment.getProperty(PROPERTY_NAMES, DEFAULT_PROPERTY_NAME).split(",")) { + String trimmed = propertyName.trim(); + if (!trimmed.isEmpty()) { + propertyNames.add(trimmed); + } + } + if (propertyNames.isEmpty()) { + propertyNames.add(DEFAULT_PROPERTY_NAME); + } + return propertyNames; + } + + /** + * The first of the configured properties asking for an embedded server, or null when none is. + * + *

The properties are read in the order they were named, so an application publishing the + * same URL into several of them settles which one describes the server by the order it listed + * them rather than by which happened to be looked at first.

+ */ + private Matcher firstAskingForEmbedded(ConfigurableEnvironment environment, Set propertyNames) { + for (String propertyName : propertyNames) { + String url = environment.getProperty(propertyName); + if (url != null) { + Matcher matcher = EMBEDDED_URL.matcher(url.trim()); + if (matcher.matches()) { + return matcher; + } + } + } + return null; + } + + /** + * The port the URL asked for, or one offset by however far the server port has moved, so two + * applications that did not name a port run side by side without colliding. + */ + private int resolvePort(ConfigurableEnvironment environment, String urlPort) { + if (urlPort != null && !urlPort.isEmpty()) { + return Integer.parseInt(urlPort); + } + int serverPort = Integer.parseInt(environment.getProperty("server.port", String.valueOf(DEFAULT_SERVER_PORT))); + return serverPort == 0 ? DEFAULT_PORT : DEFAULT_PORT + (serverPort - DEFAULT_SERVER_PORT); + } + + /** + * The database the URL named, which is the application's own configuration and the only place + * one is written down now that the URL is what asks for a server at all. + */ + private String resolveDatabase(String urlDatabase) { + return urlDatabase == null || urlDatabase.isEmpty() ? DEFAULT_DATABASE : urlDatabase; + } + +} diff --git a/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoLifecycle.java b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoLifecycle.java new file mode 100644 index 00000000000..5ac83f11e4c --- /dev/null +++ b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoLifecycle.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongodb.embedded; + +import org.springframework.context.SmartLifecycle; + +/** + * Stops the embedded server while the process is checkpointed and starts it again when the + * process is restored. + * + *

CRaC refuses to checkpoint a process that holds an open socket, and the in-memory + * backend holds several: a listening socket, the event loops behind it, and the server side + * of every connection the driver has open. Without this, an application using an embedded + * MongoDB cannot be checkpointed at all. + * + *

Spring stops {@link SmartLifecycle} beans before the checkpoint and starts them again + * after the restore, so taking part costs no dependency on {@code org.crac}. It also means + * the same bean covers an ordinary shutdown. + * + * @author Grails + * @since 8.0 + */ +public class EmbeddedMongoLifecycle implements SmartLifecycle { + + public static final String BEAN_NAME = "embeddedMongoLifecycle"; + + /** + * Spring starts in ascending phase order and stops in descending, so a phase below + * everything else makes the server the first thing up and the last thing down. It has to + * outlive the datastore that talks to it, which uses + * {@code MongoDatastore.LIFECYCLE_PHASE}, itself below the web server's. + */ + public static final int PHASE = -2000; + + private final RunningEmbeddedMongo running; + + /** The server is already listening by the time this is constructed. */ + private volatile boolean started = true; + + public EmbeddedMongoLifecycle(RunningEmbeddedMongo running) { + this.running = running; + } + + @Override + public void start() { + if (!this.started) { + this.running.restart(); + this.started = true; + } + } + + @Override + public void stop() { + if (this.started) { + this.running.stop(); + this.started = false; + } + } + + @Override + public boolean isRunning() { + return this.started; + } + + @Override + public int getPhase() { + return PHASE; + } +} diff --git a/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoSettings.java b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoSettings.java new file mode 100644 index 00000000000..5c49f2e6f98 --- /dev/null +++ b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoSettings.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongodb.embedded; + +/** + * What to start a server with. Not every backend honours every setting; one that cannot + * says so rather than starting a server that quietly behaves differently than asked. + * + * @author Grails + * @since 8.0 + */ +public final class EmbeddedMongoSettings { + + private final int port; + + private final String version; + + private final String databaseDir; + + public EmbeddedMongoSettings(int port, String version, String databaseDir) { + this.port = port; + this.version = version; + this.databaseDir = databaseDir; + } + + /** + * @return the port to bind + */ + public int getPort() { + return this.port; + } + + /** + * @return the requested server version, or null for the backend default + */ + public String getVersion() { + return this.version; + } + + /** + * @return where the data should be kept, or null to discard it when the server stops + */ + public String getDatabaseDir() { + return this.databaseDir; + } + + /** + * @return whether the data is meant to outlive the server + */ + public boolean isPersistent() { + return this.databaseDir != null && !this.databaseDir.isEmpty(); + } +} diff --git a/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/FlapdoodleMongoBackend.java b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/FlapdoodleMongoBackend.java new file mode 100644 index 00000000000..c3c6d84f7e8 --- /dev/null +++ b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/FlapdoodleMongoBackend.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongodb.embedded; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import de.flapdoodle.embed.mongo.commands.MongodArguments; +import de.flapdoodle.embed.mongo.commands.ServerAddress; +import de.flapdoodle.embed.mongo.config.Net; +import de.flapdoodle.embed.mongo.distribution.Version; +import de.flapdoodle.embed.mongo.transitions.ImmutableMongod; +import de.flapdoodle.embed.mongo.transitions.Mongod; +import de.flapdoodle.embed.mongo.transitions.RunningMongodProcess; +import de.flapdoodle.embed.mongo.types.DatabaseDir; +import de.flapdoodle.reverse.TransitionWalker; +import de.flapdoodle.reverse.transitions.Start; + +import org.springframework.util.ClassUtils; + +/** + * Runs a real mongod as a child process using Flapdoodle Embedded MongoDB, which + * downloads a genuine MongoDB binary on first use and caches it under + * {@code ~/.embedmongo}. Because it is actual MongoDB, transactions, change streams and + * {@code $text} all behave as they do in production, and the data can be kept between + * runs. + * + *

Flapdoodle is not a dependency of this module: it pulls in jgrapht, which is offered + * under LGPL-2.1 or EPL-2.0, and an Apache release should not require it. An application + * that wants a real mongod adds {@code de.flapdoodle.embed:de.flapdoodle.embed.mongo} + * itself, the same way an application picks its own SQL database driver. Every reference + * to flapdoodle is therefore confined to methods that {@link #isAvailable()} guards. + * + * @author Grails + * @since 8.0 + */ +public class FlapdoodleMongoBackend implements EmbeddedMongoBackend { + + public static final String NAME = "flapdoodle"; + + /** + * Package-private, and a compile-time constant, so that {@link EmbeddedMongoInitializer} can ask + * whether flapdoodle is present without naming this class in a way that would load it. + */ + static final String MONGOD_CLASS = "de.flapdoodle.embed.mongo.transitions.Mongod"; + + private static final String DEFAULT_VERSION = "V8_0"; + + @Override + public String getName() { + return NAME; + } + + @Override + public boolean isAvailable() { + return ClassUtils.isPresent(MONGOD_CLASS, getClass().getClassLoader()); + } + + @Override + public RunningEmbeddedMongo start(EmbeddedMongoSettings settings) { + String versionName = settings.getVersion() != null ? settings.getVersion() : DEFAULT_VERSION; + + Version.Main version; + try { + version = Version.Main.valueOf(versionName); + } + catch (IllegalArgumentException ex) { + throw new IllegalStateException(EmbeddedMongoInitializer.VERSION + "=" + versionName + + " is not a Version.Main constant. Use a name such as V8_0 or V7_0.", ex); + } + + ImmutableMongod mongod = Mongod.instance() + .withNet(Start.to(Net.class).initializedWith(Net.of("localhost", settings.getPort(), false))); + if (settings.isPersistent()) { + mongod = persistentIn(mongod, settings.getDatabaseDir()); + } + return new RunningMongod(mongod, version); + } + + /** + * Flapdoodle otherwise stores the database under a temp path it deletes on shutdown, + * and its default arguments turn off syncing to disc, so both have to change before + * anything written here outlives the process. + */ + private ImmutableMongod persistentIn(ImmutableMongod mongod, String databaseDir) { + Path path; + try { + path = Files.createDirectories(Paths.get(databaseDir).toAbsolutePath()); + } + catch (IOException ex) { + throw new IllegalStateException("Could not create the embedded MongoDB directory " + databaseDir, ex); + } + return mongod + .withDatabaseDir(Start.to(DatabaseDir.class).initializedWith(DatabaseDir.of(path))) + .withMongodArguments(Start.to(MongodArguments.class) + .initializedWith(MongodArguments.defaults().withUseDefaultSyncDelay(true))); + } + + private static final class RunningMongod implements RunningEmbeddedMongo { + + /** Held so {@link #restart()} can start the same mongod again after a CRaC restore. */ + private final ImmutableMongod mongod; + + private final Version.Main version; + + private volatile TransitionWalker.ReachedState running; + + private final ServerAddress address; + + private RunningMongod(ImmutableMongod mongod, Version.Main version) { + this.mongod = mongod; + this.version = version; + this.running = mongod.start(version); + this.address = this.running.current().getServerAddress(); + } + + @Override + public String getHost() { + return this.address.getHost(); + } + + @Override + public int getPort() { + return this.address.getPort(); + } + + @Override + public void stop() { + this.running.close(); + } + + /** + * The replacement binds the same port because {@code Net} was fixed when the server + * was first configured. Only a persistent {@code database-dir} carries data across; + * mongod is a separate process, so a checkpoint image does not contain it. + */ + @Override + public void restart() { + this.running = this.mongod.start(this.version); + } + } +} diff --git a/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/InMemoryMongoBackend.java b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/InMemoryMongoBackend.java new file mode 100644 index 00000000000..4f5892109eb --- /dev/null +++ b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/InMemoryMongoBackend.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongodb.embedded; + +import de.bwaldvogel.mongo.MongoServer; +import de.bwaldvogel.mongo.backend.memory.MemoryBackend; + +import org.springframework.util.ClassUtils; + +/** + * Runs MongoDB inside this JVM using mongo-java-server, which reimplements the MongoDB + * wire protocol in Java. It starts in milliseconds, downloads nothing and needs no + * MongoDB installation, which is why it is the default. + * + *

Being a reimplementation rather than mongod, it does not support transactions, + * change streams, {@code $text} or some {@code $expr} operators, and it cannot persist + * anything. An application that needs those should add flapdoodle and let + * {@link FlapdoodleMongoBackend} run a real mongod instead. + * + * @author Grails + * @since 8.0 + */ +public class InMemoryMongoBackend implements EmbeddedMongoBackend { + + public static final String NAME = "in-memory"; + + private static final String SERVER_CLASS = "de.bwaldvogel.mongo.MongoServer"; + + @Override + public String getName() { + return NAME; + } + + @Override + public boolean isAvailable() { + return ClassUtils.isPresent(SERVER_CLASS, getClass().getClassLoader()); + } + + @Override + public RunningEmbeddedMongo start(EmbeddedMongoSettings settings) { + if (settings.isPersistent()) { + throw new IllegalStateException("The " + NAME + " backend keeps everything in memory and cannot honour " + + EmbeddedMongoInitializer.DATABASE_DIR + ". Add " + + "de.flapdoodle.embed:de.flapdoodle.embed.mongo to run a real mongod that can, or remove the " + + "directory to accept a database that is discarded when the server stops."); + } + + MemoryBackend backend = new RetainingMemoryBackend(); + MongoServer server = new MongoServer(backend); + server.bind("localhost", settings.getPort()); + return new RunningInMemoryMongo(backend, server); + } + + /** + * A backend that keeps its collections when the server it is attached to shuts down. + * + *

{@code MongoServer.shutdownNow()} closes the backend, and + * {@code AbstractMongoBackend.close()} clears every database. That is right for a server + * that is finished with, but {@link RunningInMemoryMongo#restart()} shuts the server down + * only to release its sockets for a CRaC checkpoint and then binds a new one onto the + * same data. Without this, a restored process comes back with an empty database. + * + *

Nothing is leaked by not clearing: the data is unreachable once the last server + * using it is gone, and the JVM is exiting in the ordinary shutdown case. + */ + private static final class RetainingMemoryBackend extends MemoryBackend { + + @Override + public void close() { + // Deliberately empty; see the class comment. + } + } + + private static final class RunningInMemoryMongo implements RunningEmbeddedMongo { + + /** + * Held so {@link #restart()} can bind a new server onto the data that is already + * there. The data lives on the heap, so a CRaC checkpoint image preserves it and a + * restored process comes back with the collections it had. + */ + private final MemoryBackend backend; + + private volatile MongoServer server; + + /** + * The port that was actually bound, which is not the requested one when that was 0. + * Restarting reuses it so the url published into the environment stays correct. + */ + private final int port; + + private RunningInMemoryMongo(MemoryBackend backend, MongoServer server) { + this.backend = backend; + this.server = server; + this.port = server.getLocalAddress().getPort(); + } + + @Override + public String getHost() { + return "localhost"; + } + + @Override + public int getPort() { + return this.port; + } + + @Override + public void stop() { + this.server.shutdownNow(); + } + + @Override + public void restart() { + MongoServer restarted = new MongoServer(this.backend); + restarted.bind("localhost", this.port); + this.server = restarted; + } + } +} diff --git a/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/RunningEmbeddedMongo.java b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/RunningEmbeddedMongo.java new file mode 100644 index 00000000000..77f02b3d4e7 --- /dev/null +++ b/grails-data-mongodb/embedded/src/main/java/org/grails/datastore/gorm/mongodb/embedded/RunningEmbeddedMongo.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongodb.embedded; + +/** + * A started embedded MongoDB server. + * + * @author Grails + * @since 8.0 + */ +public interface RunningEmbeddedMongo { + + /** + * @return the host the server is listening on + */ + String getHost(); + + /** + * @return the port the server is listening on, which is the port that was actually + * bound rather than the one that was requested + */ + int getPort(); + + /** + * Stops the server. Called from a JVM shutdown hook, so it must not throw. + */ + void stop(); + + /** + * Binds the server again on the port it was already using, after {@link #stop()}. + * + *

This exists for CRaC. A checkpoint refuses to run while the process holds an open + * socket, and an in-JVM server holds a listening socket, its event loops, and the + * server side of every connection. Stopping before the checkpoint and starting again + * after the restore is what lets the process be snapshotted at all. + * + *

Whether data survives is the backend's business. The in-memory backend keeps its + * data on the heap, which the checkpoint image preserves, so it rebinds a new server + * onto the same backend. Flapdoodle keeps whatever is in its {@code database-dir} and + * loses the rest, since the {@code mongod} process is not part of the image. + */ + void restart(); +} diff --git a/grails-data-mongodb/embedded/src/main/resources/META-INF/spring.factories b/grails-data-mongodb/embedded/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000000..f3382303219 --- /dev/null +++ b/grails-data-mongodb/embedded/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.context.ApplicationContextInitializer=\ +org.grails.datastore.gorm.mongodb.embedded.EmbeddedMongoInitializer diff --git a/grails-data-mongodb/embedded/src/test/groovy/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoInitializerSpec.groovy b/grails-data-mongodb/embedded/src/test/groovy/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoInitializerSpec.groovy new file mode 100644 index 00000000000..e54e29758ee --- /dev/null +++ b/grails-data-mongodb/embedded/src/test/groovy/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoInitializerSpec.groovy @@ -0,0 +1,513 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongodb.embedded + +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients + +import org.bson.Document + +import org.springframework.context.aot.AbstractAotProcessor +import org.springframework.context.support.GenericApplicationContext +import org.springframework.core.env.MapPropertySource + +import spock.lang.Specification +import spock.lang.TempDir + +import java.nio.file.Path + +/** + * Exercises both backends against real servers. Servers started here are reaped by the + * shutdown hook the initializer registers, when this JVM exits. + */ +class EmbeddedMongoInitializerSpec extends Specification { + + @TempDir + Path temp + + + void 'a url naming a host is left to the driver'() { + given: 'what a production environment configures' + GenericApplicationContext context = contextWith([ + 'grails.mongodb.url': 'mongodb://localhost:27017/bookstore', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: 'nothing was started and the url was not touched, the way an application with h2 on ' + + 'the classpath and a PostgreSQL url never starts H2' + context.environment.getProperty('grails.mongodb.url') == 'mongodb://localhost:27017/bookstore' + context.environment.propertySources.every { it.name != 'embeddedMongoDB' } + } + + void 'nothing is started when no url is configured at all'() { + given: + GenericApplicationContext context = contextWith([:]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: + !context.environment.getProperty(EmbeddedMongoInitializer.DEFAULT_PROPERTY_NAME) + context.environment.propertySources.every { it.name != 'embeddedMongoDB' } + } + + void 'the in-memory backend serves a real MongoDB connection'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27981/bookstore', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + String url = context.environment.getProperty('grails.mongodb.url') + + then: 'the database name came from the url that asked for the server' + url == 'mongodb://localhost:27981/bookstore' + + and: 'a driver can round-trip a document through it' + roundTrip(url) == 'in-memory' + } + + void 'the flapdoodle backend serves a real MongoDB connection'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): FlapdoodleMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27982/bookstore', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + String url = context.environment.getProperty('grails.mongodb.url') + + then: + url == 'mongodb://localhost:27982/bookstore' + roundTrip(url) == 'flapdoodle' + } + + void 'flapdoodle is preferred when both backends are on the classpath'() { + given: 'no backend is named, and this module has both available in tests' + GenericApplicationContext context = contextWith([ + 'grails.mongodb.url': 'mongodb://embedded:27983', + ]) + + when: 'adding flapdoodle is the opt-in for a real mongod' + new EmbeddedMongoInitializer().initialize(context) + + then: 'a real mongod answers, which the in-memory backend could not do for a transaction' + context.environment.getProperty('grails.mongodb.url') == 'mongodb://localhost:27983/test' + } + + void 'a url that names no port is offset by however far the server port has moved'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'server.port' : '9050', + 'grails.mongodb.url' : 'mongodb://embedded/bookstore', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: 'so two applications that did not name a port do not collide' + context.environment.getProperty('grails.mongodb.url') == 'mongodb://localhost:27987/bookstore' + } + + void 'credentials copied from a real url do not stop it being recognised'() { + given: 'there is nothing to authenticate against, but the url still asks for embedded' + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://user:secret@embedded:28002/bookstore', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: + context.environment.getProperty('grails.mongodb.url') == 'mongodb://localhost:28002/bookstore' + } + + void 'the url is published into every configured property'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND) : InMemoryMongoBackend.NAME, + (EmbeddedMongoInitializer.PROPERTY_NAMES): 'grails.mongodb.url, spring.data.mongodb.uri', + 'grails.mongodb.url' : 'mongodb://embedded:27984/bookstore', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: + String expected = 'mongodb://localhost:27984/bookstore' + context.environment.getProperty('grails.mongodb.url') == expected + context.environment.getProperty('spring.data.mongodb.uri') == expected + } + + void 'a second context reuses the server the first one started'() { + given: + GenericApplicationContext first = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27985/bookstore', + ]) + new EmbeddedMongoInitializer().initialize(first) + + and: + GenericApplicationContext restarted = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27985/bookstore', + ]) + + when: 'the port is already owned, as it is after a devtools restart' + new EmbeddedMongoInitializer().initialize(restarted) + + then: + restarted.environment.getProperty('grails.mongodb.url') == 'mongodb://localhost:27985/bookstore' + } + + void 'the in-memory backend refuses to pretend it can persist'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND) : InMemoryMongoBackend.NAME, + (EmbeddedMongoInitializer.DATABASE_DIR): temp.resolve('data').toString(), + 'grails.mongodb.url' : 'mongodb://embedded:27986/bookstore', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: 'it names the backend that can, rather than silently discarding the data' + IllegalStateException e = thrown() + e.message.contains('cannot honour') + e.message.contains('flapdoodle') + } + + void 'a port held by something other than an embedded MongoDB is never reused'() { + given: 'an unrelated service holding the port, on the address a backend binds' + ServerSocket intruder = new ServerSocket(27990, 1, InetAddress.getByName('localhost')) + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27990/bookstore', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: 'it fails rather than publishing a MongoDB url pointing at that service' + IllegalStateException e = thrown() + e.message.contains('something else may already be using') + + and: 'the url still asks for a server rather than naming that one' + context.environment.getProperty('grails.mongodb.url') == 'mongodb://embedded:27990/bookstore' + context.environment.propertySources.every { it.name != 'embeddedMongoDB' } + + cleanup: + intruder.close() + } + + void 'the initializer is created without flapdoodle on the classpath'() { + given: 'flapdoodle is compileOnly, so an application that has not added it sees this' + ClassLoader withoutFlapdoodle = hiding('de.flapdoodle.') + + when: 'the backends are worked out the way the public constructor works them out' + List backends = + EmbeddedMongoInitializer.defaultBackends(withoutFlapdoodle) + + then: 'holding the flapdoodle backend would load it, and loading it resolves the types its ' + + 'methods name -- so merely offering it threw NoClassDefFoundError and no embedded ' + + 'server could start at all' + backends*.name == [InMemoryMongoBackend.NAME] + } + + void 'flapdoodle is offered when it is on the classpath'() { + expect: + EmbeddedMongoInitializer.defaultBackends(getClass().classLoader)*.name == + [FlapdoodleMongoBackend.NAME, InMemoryMongoBackend.NAME] + } + + void 'asking for flapdoodle without it on the classpath says so'() { + given: + GenericApplicationContext context = new GenericApplicationContext() + context.environment.propertySources.addFirst(new MapPropertySource('test', [ + 'embedded.mongodb.backend': FlapdoodleMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27992/bookstore', + ])) + + when: 'the backend it asked for is not among the ones offered' + new EmbeddedMongoInitializer([new InMemoryMongoBackend()]).initialize(context) + + then: 'which is a missing library rather than a name that means nothing' + IllegalStateException e = thrown() + e.message.contains('not on the classpath') + + cleanup: + context.close() + } + + /** A loader that cannot see the named package, standing in for it not being on the classpath. */ + private ClassLoader hiding(String packagePrefix) { + new ClassLoader(getClass().classLoader) { + @Override + Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (name.startsWith(packagePrefix)) { + throw new ClassNotFoundException(name) + } + super.loadClass(name, resolve) + } + } + } + + void 'a known backend whose library is missing names the ones that are left'() { + given: 'mongo-java-server excluded from an application that still asked for it' + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded', + ]) + + when: + new EmbeddedMongoInitializer([new MissingLibraryBackend(InMemoryMongoBackend.NAME), + new FlapdoodleMongoBackend()]).initialize(context) + + then: 'the message points at the library, and at what could be used instead' + IllegalStateException e = thrown() + e.message.contains('not on the classpath') + e.message.contains('choose one of [flapdoodle]') + } + + void 'no backend at all is reported as the missing dependency it is'() { + given: 'both libraries excluded, so nothing can serve the url this was asked to publish' + GenericApplicationContext context = contextWith([ + 'grails.mongodb.url' : 'mongodb://embedded', + ]) + + when: 'no backend is named either, so this is the fall through rather than a bad choice' + new EmbeddedMongoInitializer([]).initialize(context) + + then: + IllegalStateException e = thrown() + e.message.contains('no embedded MongoDB backend is on the classpath') + e.message.contains('de.bwaldvogel:mongo-java-server') + } + + void 'an unknown backend is reported by name'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): 'sqlite', + 'grails.mongodb.url' : 'mongodb://embedded:27989/bookstore', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: + IllegalStateException e = thrown() + e.message.contains('is not a known backend') + } + + void 'the MongoDB port moves with the server port so two applications run side by side'() { + given: 'the second application on a machine, which moved its own port to start at all' + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'server.port' : '9055', + 'grails.mongodb.url' : 'mongodb://embedded', + ]) + + when: 'no MongoDB port is configured, so it follows' + new EmbeddedMongoInitializer().initialize(context) + + then: '27017 offset by however far 8080 moved' + context.environment.getProperty('grails.mongodb.url') == 'mongodb://localhost:27992/test' + } + + void 'a url that names no database leaves the default in place'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27987', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: 'there was no database name to preserve, rather than an empty one to publish' + context.environment.getProperty('grails.mongodb.url') == 'mongodb://localhost:27987/test' + } + + void 'a property-names list of nothing but separators still publishes somewhere'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND) : InMemoryMongoBackend.NAME, + (EmbeddedMongoInitializer.PROPERTY_NAMES): ' , ', + 'grails.mongodb.url' : 'mongodb://embedded:27988/bookstore', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: 'a started server no application can reach is worse than falling back to the default' + context.environment.getProperty('grails.mongodb.url') == 'mongodb://localhost:27988/bookstore' + } + + void 'a setting left blank is a setting that was not made'() { + given: 'the keys are present with nothing after them, which is what empty yaml entries give' + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND) : '', + 'server.port' : '9064', + 'grails.mongodb.url' : 'mongodb://embedded', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: 'rather than an unknown backend named the empty string, a port that will not parse, ' + + 'or a database with no name' + context.environment.getProperty('grails.mongodb.url') == 'mongodb://localhost:28001/test' + } + + void 'a backend whose library is missing is passed over rather than chosen'() { + given: 'flapdoodle offered first, as it always is, but excluded by the application' + GenericApplicationContext context = contextWith([ + 'grails.mongodb.url' : 'mongodb://embedded:28000', + ]) + + when: 'no backend is named, so the first one that can actually run is used' + new EmbeddedMongoInitializer([new MissingLibraryBackend(FlapdoodleMongoBackend.NAME), + new InMemoryMongoBackend()]).initialize(context) + + then: + context.environment.getProperty('grails.mongodb.url') == 'mongodb://localhost:28000/test' + } + + void 'a MongoDB version flapdoodle does not know is reported before anything is downloaded'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): FlapdoodleMongoBackend.NAME, + (EmbeddedMongoInitializer.VERSION): 'V9_9', + 'grails.mongodb.url' : 'mongodb://embedded:27993', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: 'the constant it needed, rather than a download that fails halfway' + IllegalStateException e = thrown() + e.message.contains('is not a Version.Main constant') + e.message.contains('V8_0') + + and: + !listening(27993) + } + + void 'a database directory that cannot be created is reported by path'() { + given: 'a file where the directory should be, which is how a mistyped path usually looks' + Path file = temp.resolve('prodDb') + file.toFile().text = 'not a directory' + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND) : FlapdoodleMongoBackend.NAME, + (EmbeddedMongoInitializer.DATABASE_DIR): file.toString(), + 'grails.mongodb.url' : 'mongodb://embedded:27998', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: 'rather than a mongod that starts and writes where nobody looks' + IllegalStateException e = thrown() + e.message.contains('Could not create the embedded MongoDB directory') + e.message.contains(file.toString()) + + and: + !listening(27998) + } + + /** A backend whose library an application excluded, which only its absence distinguishes. */ + private static class MissingLibraryBackend implements EmbeddedMongoBackend { + + private final String name + + MissingLibraryBackend(String name) { + this.name = name + } + + @Override + String getName() { + name + } + + @Override + boolean isAvailable() { + false + } + + @Override + RunningEmbeddedMongo start(EmbeddedMongoSettings settings) { + throw new UnsupportedOperationException('not available') + } + } + + private static String roundTrip(String url) { + try (MongoClient client = MongoClients.create(url)) { + def collection = client.getDatabase('bookstore').getCollection('probe') + collection.insertOne(new Document('backend', url.contains('27981') ? 'in-memory' : 'flapdoodle')) + collection.find().first().getString('backend') + } + } + + void 'nothing is started while bean definitions are being generated'() { + given: 'a configuration that would otherwise start a server, on a port no other feature ' + + 'here uses -- servers started in this spec outlive the feature that started them' + assert !listening(27991) + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27991/bookstore', + ]) + System.setProperty(AbstractAotProcessor.AOT_PROCESSING, 'true') + + when: + new EmbeddedMongoInitializer().initialize(context) + + then: 'generation reads definitions rather than running them, and a server that started ' + + 'would listen on a non-daemon thread and hang the build that started it' + context.environment.getProperty('grails.mongodb.url') == 'mongodb://embedded:27991/bookstore' + context.environment.propertySources.every { it.name != 'embeddedMongoDB' } + + and: 'nothing is listening on the port it was told to use' + !listening(27991) + + cleanup: + System.clearProperty(AbstractAotProcessor.AOT_PROCESSING) + } + + private static boolean listening(int port) { + try { + new Socket('localhost', port).withCloseable { true } + } + catch (IOException ignored) { + false + } + } + + private static GenericApplicationContext contextWith(Map properties) { + GenericApplicationContext context = new GenericApplicationContext() + context.environment.propertySources.addFirst(new MapPropertySource('test', properties)) + context + } +} diff --git a/grails-data-mongodb/embedded/src/test/groovy/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoLifecycleSpec.groovy b/grails-data-mongodb/embedded/src/test/groovy/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoLifecycleSpec.groovy new file mode 100644 index 00000000000..2fe46082b0e --- /dev/null +++ b/grails-data-mongodb/embedded/src/test/groovy/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoLifecycleSpec.groovy @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongodb.embedded + +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients + +import org.bson.Document + +import org.springframework.context.support.GenericApplicationContext +import org.springframework.core.env.MapPropertySource + +import spock.lang.Specification +import spock.lang.TempDir +import spock.util.concurrent.PollingConditions + +import java.nio.file.Path + +/** + * Exercises the stop and restart that a CRaC checkpoint and restore drive. + * + *

Stopping is what releases the sockets that would otherwise refuse the checkpoint, and + * starting again has to leave the application looking at the same database on the same + * port -- a restored process that silently comes back empty is a worse failure than one + * that could not be checkpointed at all. + */ +class EmbeddedMongoLifecycleSpec extends Specification { + + @TempDir + Path temp + + PollingConditions conditions = new PollingConditions(timeout: 20, delay: 0.1) + + void 'the lifecycle of the server it started is managed by the application context'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND): InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27994', + ]) + + when: + new EmbeddedMongoInitializer().initialize(context) + EmbeddedMongoLifecycle lifecycle = context.beanFactory + .getBean(EmbeddedMongoLifecycle.BEAN_NAME, EmbeddedMongoLifecycle) + + then: 'the server is already listening by the time the bean exists' + lifecycle.running + + and: 'it starts before, and stops after, the datastore that talks to it' + lifecycle.phase == EmbeddedMongoLifecycle.PHASE + lifecycle.phase < 0 + + cleanup: + lifecycle?.stop() + } + + void 'the in-memory backend keeps its data across the stop a checkpoint needs'() { + given: 'a server holding a document, as an application would before it is checkpointed' + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND) : InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27995/bookstore', + ]) + new EmbeddedMongoInitializer().initialize(context) + String url = context.environment.getProperty('grails.mongodb.url') + write(url, 'Groovy in Action') + + and: + EmbeddedMongoLifecycle lifecycle = context.beanFactory + .getBean(EmbeddedMongoLifecycle.BEAN_NAME, EmbeddedMongoLifecycle) + + when: 'the checkpoint stops it, because CRaC refuses to snapshot a process holding sockets' + lifecycle.stop() + + then: + !lifecycle.running + conditions.eventually { assert !listening(27995) } + + when: 'the restore starts it again' + lifecycle.start() + + then: 'it is back on the port the published url already names' + lifecycle.running + conditions.eventually { assert listening(27995) } + + and: 'holding what it held, rather than the empty database a cleared backend leaves' + titles(url) == ['Groovy in Action'] + + cleanup: + lifecycle?.stop() + } + + void 'stopping and starting more than once does the work only once'() { + given: + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND) : InMemoryMongoBackend.NAME, + 'grails.mongodb.url' : 'mongodb://embedded:27996/bookstore', + ]) + new EmbeddedMongoInitializer().initialize(context) + String url = context.environment.getProperty('grails.mongodb.url') + EmbeddedMongoLifecycle lifecycle = context.beanFactory + .getBean(EmbeddedMongoLifecycle.BEAN_NAME, EmbeddedMongoLifecycle) + + when: 'a context that is already stopped is stopped again' + lifecycle.stop() + lifecycle.stop() + + then: + !lifecycle.running + + when: 'and a running one is started again, which binding a second server would fail' + lifecycle.start() + lifecycle.start() + + then: + lifecycle.running + conditions.eventually { assert listening(27996) } + + and: 'the server still answers' + write(url, 'Making Java Groovy') + titles(url) == ['Making Java Groovy'] + + cleanup: + lifecycle?.stop() + } + + void 'the flapdoodle backend keeps a persistent database across the same stop'() { + given: 'a real mongod told to keep its data, which is how a production application runs' + String databaseDir = temp.resolve('prodDb').toString() + GenericApplicationContext context = contextWith([ + (EmbeddedMongoInitializer.BACKEND) : FlapdoodleMongoBackend.NAME, + (EmbeddedMongoInitializer.DATABASE_DIR): databaseDir, + 'grails.mongodb.url' : 'mongodb://embedded:27997/bookstore', + ]) + new EmbeddedMongoInitializer().initialize(context) + String url = context.environment.getProperty('grails.mongodb.url') + write(url, 'Grails in Action') + + and: + EmbeddedMongoLifecycle lifecycle = context.beanFactory + .getBean(EmbeddedMongoLifecycle.BEAN_NAME, EmbeddedMongoLifecycle) + + when: + lifecycle.stop() + + then: 'the mongod process is gone, so the driver has nothing to reconnect to' + conditions.eventually { assert !listening(27997) } + + and: 'and what it wrote is on disc rather than in a temp directory it deleted' + new File(databaseDir).list() + + when: + lifecycle.start() + + then: + conditions.eventually { assert listening(27997) } + titles(url) == ['Grails in Action'] + + cleanup: + lifecycle?.stop() + } + + private static void write(String url, String title) { + try (MongoClient client = MongoClients.create(url)) { + client.getDatabase('bookstore').getCollection('books').insertOne(new Document('title', title)) + } + } + + private static List titles(String url) { + try (MongoClient client = MongoClients.create(url)) { + client.getDatabase('bookstore').getCollection('books').find()*.getString('title') + } + } + + private static boolean listening(int port) { + try { + new Socket('localhost', port).withCloseable { true } + } + catch (IOException ignored) { + false + } + } + + private static GenericApplicationContext contextWith(Map properties) { + GenericApplicationContext context = new GenericApplicationContext() + context.environment.propertySources.addFirst(new MapPropertySource('test', properties)) + context + } +} diff --git a/grails-data-mongodb/embedded/src/test/groovy/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoSettingsSpec.groovy b/grails-data-mongodb/embedded/src/test/groovy/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoSettingsSpec.groovy new file mode 100644 index 00000000000..c95c832f693 --- /dev/null +++ b/grails-data-mongodb/embedded/src/test/groovy/org/grails/datastore/gorm/mongodb/embedded/EmbeddedMongoSettingsSpec.groovy @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongodb.embedded + +import spock.lang.Specification +import spock.lang.Unroll + +class EmbeddedMongoSettingsSpec extends Specification { + + void 'what a backend is asked for is what it was given'() { + when: + EmbeddedMongoSettings settings = new EmbeddedMongoSettings(27017, 'V8_0', '/var/data/mongo') + + then: + settings.port == 27017 + settings.version == 'V8_0' + settings.databaseDir == '/var/data/mongo' + } + + void 'a version the application did not choose is left to the backend'() { + expect: 'null rather than a default here, so each backend picks one it can actually run' + new EmbeddedMongoSettings(27017, null, null).version == null + } + + @Unroll + void 'a database directory of #databaseDir means persistent = #persistent'() { + expect: 'an empty directory is no directory, so a backend that cannot persist does not refuse' + new EmbeddedMongoSettings(27017, null, databaseDir).persistent == persistent + + where: + databaseDir || persistent + null || false + '' || false + './prodDb' || true + } +} diff --git a/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/GrailsDataMongoDB.java b/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/GrailsDataMongoDB.java index 254863aac79..5990061ac40 100644 --- a/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/GrailsDataMongoDB.java +++ b/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/GrailsDataMongoDB.java @@ -34,6 +34,13 @@ @Singleton public class GrailsDataMongoDB extends GormOneOfFeature { + /** + * The host that asks grails-data-mongodb-embedded for a server rather than naming one to + * reach, so an environment says which database it wants in the one place it already says + * where the database is. + */ + private static final String EMBEDDED_HOST = "mongodb://embedded/"; + private final TestContainers testContainers; public GrailsDataMongoDB(TestContainers testContainers) { @@ -67,11 +74,31 @@ public void processSelectedFeatures(FeatureContext featureContext) { public void apply(GeneratorContext generatorContext) { applyDefaultGormConfig(generatorContext.getConfiguration()); Map config = generatorContext.getConfiguration(); - config.put("grails.mongodb.url", "mongodb://${MONGO_HOST:localhost}:${MONGO_PORT:27017}/foo"); + config.put("grails.mongodb.url", MongoFeature.externalUrl(MongoFeature.PROD_DATABASE)); generatorContext.addDependency(Dependency.builder() .groupId("org.apache.grails") .artifactId("grails-data-mongodb") .implementation()); + applyEmbeddedMongo(generatorContext, config); + } + + /** + * Without this a generated application only starts when a MongoDB happens to be listening on + * the url above, so it is wired in by default rather than offered as a separate feature. + * + *

Development and test name the embedded server where they would name a host, the way a + * generated JPA application names an in-memory H2 there. Production keeps the url above, so + * deploying with MONGO_HOST and MONGO_PORT set reaches that database. An application that + * wants an embedded one in production says so the same way, by naming it in that url. + */ + private void applyEmbeddedMongo(GeneratorContext generatorContext, Map config) { + generatorContext.addDependency(Dependency.builder() + .groupId("org.apache.grails") + .artifactId("grails-data-mongodb-embedded") + .implementation()); + + config.put("environments.development.grails.mongodb.url", EMBEDDED_HOST + MongoFeature.DEV_DATABASE); + config.put("environments.test.grails.mongodb.url", EMBEDDED_HOST + MongoFeature.TEST_DATABASE); } @Override diff --git a/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/MongoFeature.java b/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/MongoFeature.java index 0c93dc147de..8a184e5309e 100644 --- a/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/MongoFeature.java +++ b/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/MongoFeature.java @@ -21,8 +21,22 @@ import org.grails.forge.feature.Feature; import org.grails.forge.feature.FeatureContext; +/** + * Common ground for the features that point a generated application at a MongoDB. + * + *

The database names below are the ones a generated application already uses for its SQL + * database, so an application with both does not call them different things, and are held here + * rather than in each feature so that two MongoDB features cannot drift apart. {@code + * GrailsDataMongoDB} uses them without extending this, since it is a GORM feature first. + */ public abstract class MongoFeature implements Feature { + static final String DEV_DATABASE = "devDb"; + + static final String TEST_DATABASE = "testDb"; + + static final String PROD_DATABASE = "prodDb"; + private final TestContainers testContainers; public MongoFeature(TestContainers testContainers) { @@ -36,4 +50,15 @@ public void processSelectedFeatures(FeatureContext featureContext) { } } + /** + * The url of a MongoDB the application has to reach, which deploying with {@code MONGO_HOST} + * and {@code MONGO_PORT} set points at that server. + * + * @param database the database to use on it + * @return the connection url + */ + static String externalUrl(String database) { + return "mongodb://${MONGO_HOST:localhost}:${MONGO_PORT:27017}/" + database; + } + } diff --git a/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/MongoSync.java b/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/MongoSync.java index 6ce42eef78e..9d5f7fd09ec 100644 --- a/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/MongoSync.java +++ b/grails-forge/grails-forge-core/src/main/java/org/grails/forge/feature/database/MongoSync.java @@ -51,7 +51,11 @@ public String getDescription() { @Override public void apply(GeneratorContext generatorContext) { Map config = generatorContext.getConfiguration(); - config.put("grails.mongodb.url", "mongodb://${MONGO_HOST:localhost}:${MONGO_PORT:27017}/foo"); + // One url cannot name a database that is right in every environment, so each names its + // own on the same server, the way a generated SQL application already does. + config.put("grails.mongodb.url", externalUrl(PROD_DATABASE)); + config.put("environments.development.grails.mongodb.url", externalUrl(DEV_DATABASE)); + config.put("environments.test.grails.mongodb.url", externalUrl(TEST_DATABASE)); generatorContext.addDependency(Dependency.builder() .groupId("org.mongodb") .artifactId("mongodb-driver-sync") diff --git a/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/feature/database/GrailsDataMongoDBSpec.groovy b/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/feature/database/GrailsDataMongoDBSpec.groovy index b2a62b23d61..94d1505817c 100644 --- a/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/feature/database/GrailsDataMongoDBSpec.groovy +++ b/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/feature/database/GrailsDataMongoDBSpec.groovy @@ -56,6 +56,7 @@ class GrailsDataMongoDBSpec extends ApplicationContextSpec implements CommandOut then: template.contains("implementation \"org.apache.grails:grails-data-mongodb\"") + template.contains("implementation \"org.apache.grails:grails-data-mongodb-embedded\"") } void "test config"() { @@ -66,6 +67,29 @@ class GrailsDataMongoDBSpec extends ApplicationContextSpec implements CommandOut ctx.configuration.containsKey("grails.mongodb.url") } + void "test the embedded MongoDB is enabled for development and test only"() { + when: + GeneratorContext ctx = buildGeneratorContext(['gorm-mongodb']) + + then: 'development and test name the server where they would name a host, so the app runs ' + + 'with no MongoDB installed' + ctx.configuration.get("environments.development.grails.mongodb.url") == 'mongodb://embedded/devDb' + ctx.configuration.get("environments.test.grails.mongodb.url") == 'mongodb://embedded/testDb' + + and: 'production keeps the configured url, so deploying with MONGO_HOST set reaches that database' + !ctx.configuration.containsKey("environments.production.grails.mongodb.url") + ctx.configuration.get("grails.mongodb.url") == 'mongodb://${MONGO_HOST:localhost}:${MONGO_PORT:27017}/prodDb' + } + + void "test no initializer is copied into the generated application"() { + when: + Map output = generate(['gorm-mongodb']) + + then: 'the embedded support arrives as a dependency, not as generated source' + !output.keySet().any { it.endsWith('EmbeddedMongoConfig.groovy') } + !output.containsKey('src/main/resources/META-INF/spring.factories') + } + void "test a SQL driver combined with MongoDB still adds Hibernate 5 as the default SQL implementation"() { given: Options options = new Options(DevelopmentReloading.DEFAULT_OPTION, GormImpl.MONGODB, ServletImpl.DEFAULT_OPTION, JdkVersion.DEFAULT_OPTION) diff --git a/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/feature/database/MongoSyncSpec.groovy b/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/feature/database/MongoSyncSpec.groovy index fec83fbd4b9..60ab7825c92 100644 --- a/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/feature/database/MongoSyncSpec.groovy +++ b/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/feature/database/MongoSyncSpec.groovy @@ -21,6 +21,7 @@ package org.grails.forge.feature.database import org.grails.forge.ApplicationContextSpec import org.grails.forge.BuildBuilder +import org.grails.forge.application.generator.GeneratorContext import org.grails.forge.feature.Features import org.grails.forge.fixture.CommandOutputFixture @@ -45,6 +46,23 @@ class MongoSyncSpec extends ApplicationContextSpec implements CommandOutputFixtu features.contains("mongo-sync") } + void "test each environment gets its own database on the configured server"() { + when: + GeneratorContext ctx = buildGeneratorContext(['mongo-sync']) + + then: 'the same names a generated SQL application uses, rather than one database shared ' + + 'by every environment' + ctx.configuration.get("environments.development.grails.mongodb.url") == + 'mongodb://${MONGO_HOST:localhost}:${MONGO_PORT:27017}/devDb' + ctx.configuration.get("environments.test.grails.mongodb.url") == + 'mongodb://${MONGO_HOST:localhost}:${MONGO_PORT:27017}/testDb' + + and: 'production stays the default, so deploying with MONGO_HOST set reaches that database' + !ctx.configuration.containsKey("environments.production.grails.mongodb.url") + ctx.configuration.get("grails.mongodb.url") == + 'mongodb://${MONGO_HOST:localhost}:${MONGO_PORT:27017}/prodDb' + } + void "test mongo sync dependencies are present for gradle"() { when: String template = new BuildBuilder(beanContext) diff --git a/settings.gradle b/settings.gradle index ac25c7e025f..e5be5a71b00 100644 --- a/settings.gradle +++ b/settings.gradle @@ -393,6 +393,9 @@ project(':grails-data-mongodb-spring-boot').projectDir = new File(settingsDir, ' include 'grails-data-mongodb-spring-data' project(':grails-data-mongodb-spring-data').projectDir = new File(settingsDir, 'grails-data-mongodb/spring-data') +include 'grails-data-mongodb-embedded' +project(':grails-data-mongodb-embedded').projectDir = new File(settingsDir, 'grails-data-mongodb/embedded') + include 'grails-data-mongodb' project(':grails-data-mongodb').projectDir = new File(settingsDir, 'grails-data-mongodb/grails-plugin')