diff --git a/build.gradle b/build.gradle index ce00bbd208b..61f25361f56 100644 --- a/build.gradle +++ b/build.gradle @@ -47,7 +47,14 @@ ext { // needing --rerun-tasks. Useful for repeatedly running the same test command while // chasing flaky tests across runs. doNotCacheTests = System.getenv('DO_NOT_CACHE_TESTS')?.toBoolean() - configuredTestParallel = findProperty('maxTestParallel') as Integer ?: (isCiBuild ? 4 : Runtime.runtime.availableProcessors() * 3 / 4 as int ?: 1) + // availableProcessors() reports LOGICAL processors: SMT threads on x64, and on Apple silicon + // every efficiency core as well as every performance core. Taking 3/4 of that therefore + // asked for more concurrent Grails/Spring test JVMs than the machine has real capacity for + // (each fork also holds a 1G heap, and the mongodb, redis and geb suites start a Docker + // container per fork). Half is the conservative fraction. CI keeps its existing budget, so + // this does not reduce CI fork counts - though CI forks do see fewer processors, below. + configuredTestParallel = findProperty('maxTestParallel') as Integer ?: + (isCiBuild ? 4 : Math.max(1, (Runtime.runtime.availableProcessors() / 2) as int)) excludeUnusedTransDeps = findProperty('excludeUnusedTransDeps') testProjectsStartWith = [ @@ -60,8 +67,53 @@ ext { profileProjects = [ /* Will be populated by subprojects loop below */] } +/** + * Tells a forked test JVM how many processors it may size its GC and JIT thread pools from. + * + * Every forked JVM otherwise sizes those pools for the WHOLE machine, because no fork knows the + * others exist. On a 20-processor host that is 15 ParallelGCThreads + 4 ConcGCThreads + 12 + * CICompilerCount = 31 threads per fork before a single test runs, so N concurrent forks become + * N*31 threads competing for the same cores. Gradle cannot see this: it charges each fork one + * worker lease, as though a fork were a single thread. + * + * The share is availableProcessors / maxWorkerCount, i.e. the BUILD-WIDE worker limit, because + * that - not any one task's maxParallelForks - is what bounds how many forks can run at once. + * With org.gradle.parallel=true, Test tasks from several projects execute concurrently, so a + * per-task share would still oversubscribe the machine. + * + * Supplied as an argument provider rather than through jvmArgs because several modules assign + * jvmArgs wholesale in their own build scripts, which would discard anything added here first. + */ +final class ActiveProcessorCountArgumentProvider implements CommandLineArgumentProvider { + + @Input + final int processorCount + + ActiveProcessorCountArgumentProvider(int availableProcessorCount, int maxWorkerCount) { + // Floor of 2: at ActiveProcessorCount=1 HotSpot's ergonomics select Serial GC, which + // costs more on a 1G test heap than the contention it saves. Two keeps G1 with a + // bounded worker count. + // + // That floor also means -PmaxTestParallel=1 still advertises 2 processors to the + // single fork whenever maxWorkerCount is larger than availableProcessors/2 (the + // usual local case: org.gradle.parallel=true with no org.gradle.workers.max). + // It is not a "give the one fork the whole machine" switch. + this.processorCount = Math.max(2, (availableProcessorCount / Math.max(1, maxWorkerCount)) as int) + } + + @Override + Iterable asArguments() { + ["-XX:ActiveProcessorCount=$processorCount".toString()] + } +} + subprojects { + tasks.withType(Test).configureEach { testTask -> + testTask.jvmArgumentProviders.add(new ActiveProcessorCountArgumentProvider( + Runtime.runtime.availableProcessors(), gradle.startParameter.maxWorkerCount)) + } + for (String testPrefix : testProjectsStartWith) { if (name.startsWith(testPrefix)) { rootProject.ext['testProjects'] << name diff --git a/gradle/functional-test-config.gradle b/gradle/functional-test-config.gradle index c92ccc76072..7b11bbdfa28 100644 --- a/gradle/functional-test-config.gradle +++ b/gradle/functional-test-config.gradle @@ -170,7 +170,7 @@ tasks.withType(Test).configureEach { Test task -> showStackTraces = true } if (findProperty('testForked')) { - task.maxParallelForks = configuredTestParallel + task.maxParallelForks = rootProject.ext.configuredTestParallel task.forkEvery = (findProperty('testForkEvery') ?: 0) } if (findProperty('testJvmArgs')) { diff --git a/gradle/mongodb-forked-test-config.gradle b/gradle/mongodb-forked-test-config.gradle index 77f27033227..947665619cb 100644 --- a/gradle/mongodb-forked-test-config.gradle +++ b/gradle/mongodb-forked-test-config.gradle @@ -50,7 +50,7 @@ tasks.withType(Test).configureEach { } useJUnitPlatform() - maxParallelForks = configuredTestParallel + maxParallelForks = rootProject.ext.configuredTestParallel jvmArgs = ['-Xmx768M'] addTestListener([ afterSuite: { desc, result -> @@ -67,4 +67,4 @@ tasks.withType(Test).configureEach { } // Used in the TCK test to selectively enable/disable tests systemProperty('mongodb.gorm.suite', 'true') -} \ No newline at end of file +} diff --git a/gradle/test-config.gradle b/gradle/test-config.gradle index e4ad66b9c5a..d6ea685c670 100644 --- a/gradle/test-config.gradle +++ b/gradle/test-config.gradle @@ -83,7 +83,7 @@ tasks.withType(Test).configureEach { showStackTraces = true } excludes = ['**/*TestCase.class', '**/*$*.class'] - maxParallelForks = configuredTestParallel + maxParallelForks = rootProject.ext.configuredTestParallel maxHeapSize = isCiBuild ? '768m' : '1024m' forkEvery = hasProperty('forkEveryUnitTest') ? getProperty('forkEveryUnitTest') as long : (isCiBuild ? 50 : 100) if (System.getProperty('debug.tests')) { diff --git a/grails-core/src/test/groovy/grails/util/ActiveProcessorCountForkTests.java b/grails-core/src/test/groovy/grails/util/ActiveProcessorCountForkTests.java new file mode 100644 index 00000000000..fd1cf7d537f --- /dev/null +++ b/grails-core/src/test/groovy/grails/util/ActiveProcessorCountForkTests.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package grails.util; + +import java.lang.management.ManagementFactory; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Asserts that {@code -XX:ActiveProcessorCount} from the root build's + * {@code jvmArgumentProviders} actually reaches this forked test JVM. + * Wholesale {@code jvmArgs = ...} assignments in module scripts would drop a + * flag added via {@code jvmArgs}, which is why the build uses a provider. + */ +class ActiveProcessorCountForkTests { + + private static final String FLAG_PREFIX = "-XX:ActiveProcessorCount="; + + @Test + void forkedTestJvmReceivesActiveProcessorCount() { + String flag = findActiveProcessorCountFlag(); + assertNotNull(flag, + "forked test JVM must receive -XX:ActiveProcessorCount from jvmArgumentProviders"); + int advertised = Integer.parseInt(flag.substring(FLAG_PREFIX.length())); + assertTrue(advertised >= 2, "floor is 2 so HotSpot keeps G1: " + advertised); + assertEquals(advertised, Runtime.getRuntime().availableProcessors(), + "HotSpot must honour the advertised processor count"); + } + + private static String findActiveProcessorCountFlag() { + List args = ManagementFactory.getRuntimeMXBean().getInputArguments(); + for (String arg : args) { + if (arg.startsWith(FLAG_PREFIX)) { + return arg; + } + } + return null; + } +} diff --git a/grails-data-neo4j/grails-datastore-gorm-neo4j/build.gradle b/grails-data-neo4j/grails-datastore-gorm-neo4j/build.gradle index 8267779b0a8..852ef75dc07 100644 --- a/grails-data-neo4j/grails-datastore-gorm-neo4j/build.gradle +++ b/grails-data-neo4j/grails-datastore-gorm-neo4j/build.gradle @@ -47,7 +47,7 @@ dependencies { test { useJUnitPlatform() - maxParallelForks = configuredTestParallel + maxParallelForks = rootProject.ext.configuredTestParallel forkEvery = 10 jvmArgs = ['-Xmx1028M'] diff --git a/grails-forge/build.gradle b/grails-forge/build.gradle index d262f97fab9..a149ee6e110 100644 --- a/grails-forge/build.gradle +++ b/grails-forge/build.gradle @@ -42,10 +42,32 @@ ext { // needing --rerun-tasks. Useful for repeatedly running the same test command while // chasing flaky tests across runs. doNotCacheTests = System.getenv('DO_NOT_CACHE_TESTS')?.toBoolean() + // Mirror grails-gradle: honour -PmaxTestParallel, and cap CI forks so a 4-vCPU + // runner does not inherit the local "half the logical processors" formula. + configuredTestParallel = findProperty('maxTestParallel') as Integer ?: + (isCiBuild ? 3 : Math.max(1, (Runtime.runtime.availableProcessors() / 2) as int)) +} + +/** Caps each test fork's view of the machine. See the root build.gradle for the rationale. */ +final class ActiveProcessorCountArgumentProvider implements CommandLineArgumentProvider { + + @Input + final int processorCount + + ActiveProcessorCountArgumentProvider(int availableProcessorCount, int maxWorkerCount) { + // Floor of 2 keeps G1. -PmaxTestParallel=1 still sees 2 processors when + // maxWorkerCount is the usual unrestricted local worker pool. + this.processorCount = Math.max(2, (availableProcessorCount / Math.max(1, maxWorkerCount)) as int) + } + + @Override + Iterable asArguments() { + ["-XX:ActiveProcessorCount=$processorCount".toString()] + } } allprojects { - tasks.withType(Test).configureEach { Task testTask -> + tasks.withType(Test).configureEach { testTask -> testTask.dependsOn( gradle.includedBuild('grails-core').task(':publishAllPublicationsToTestCaseMavenRepoRepository'), gradle.includedBuild('grails-gradle').task(':publishAllPublicationsToTestCaseMavenRepoRepository') @@ -55,6 +77,8 @@ allprojects { // without --rerun-tasks (and without recompiling everything else). testTask.outputs.cacheIf { !doNotCacheTests } testTask.outputs.upToDateWhen { !doNotCacheTests } + testTask.jvmArgumentProviders.add(new ActiveProcessorCountArgumentProvider( + Runtime.runtime.availableProcessors(), gradle.startParameter.maxWorkerCount)) } } @@ -72,4 +96,4 @@ apply { // we must apply the publish configuration first or the docs config will not work from layout.projectDirectory.file('gradle/publish-root-config.gradle') from layout.projectDirectory.file('gradle/gradle-wrapper-root-config.gradle') -} \ No newline at end of file +} diff --git a/grails-forge/gradle/test-config.gradle b/grails-forge/gradle/test-config.gradle index 9d4d27ddf9c..5f2df2b80b2 100644 --- a/grails-forge/gradle/test-config.gradle +++ b/grails-forge/gradle/test-config.gradle @@ -49,7 +49,7 @@ tasks.withType(Test).configureEach { jvmArgs('-Duser.country=US', '-Duser.language=en', '--add-opens', 'java.base/java.lang=ALL-UNNAMED') forkEvery = 100 - maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 + maxParallelForks = rootProject.ext.configuredTestParallel maxHeapSize = '2G' useJUnitPlatform() diff --git a/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/ActiveProcessorCountForkSpec.groovy b/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/ActiveProcessorCountForkSpec.groovy new file mode 100644 index 00000000000..f1f9ea3bfa2 --- /dev/null +++ b/grails-forge/grails-forge-core/src/test/groovy/org/grails/forge/ActiveProcessorCountForkSpec.groovy @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.forge + +import java.lang.management.ManagementFactory + +import spock.lang.Specification + +class ActiveProcessorCountForkSpec extends Specification { + + private static final String FLAG_PREFIX = '-XX:ActiveProcessorCount=' + + void 'forked test JVM receives ActiveProcessorCount from jvmArgumentProviders'() { + given: + String flag = ManagementFactory.runtimeMXBean.inputArguments.find { it.startsWith(FLAG_PREFIX) } + + expect: + flag != null + int advertised = Integer.parseInt(flag.substring(FLAG_PREFIX.length())) + advertised >= 2 + Runtime.runtime.availableProcessors() == advertised + } +} diff --git a/grails-gradle/build.gradle b/grails-gradle/build.gradle index 382c9771024..bc42c184e2c 100644 --- a/grails-gradle/build.gradle +++ b/grails-gradle/build.gradle @@ -47,7 +47,35 @@ ext { // needing --rerun-tasks. Useful for repeatedly running the same test command while // chasing flaky tests across runs. doNotCacheTests = System.getenv('DO_NOT_CACHE_TESTS')?.toBoolean() - configuredTestParallel = findProperty('maxTestParallel') as Integer ?: (isCiBuild ? 3 : Runtime.runtime.availableProcessors() * 3/4 as int ?: 1) + // Half the LOGICAL processors, not 3/4. This separate Gradle build mirrors the root + // build.gradle - see it for the full rationale. CI keeps its existing budget. + configuredTestParallel = findProperty('maxTestParallel') as Integer ?: + (isCiBuild ? 3 : Math.max(1, (Runtime.runtime.availableProcessors() / 2) as int)) +} + +/** Caps each test fork's view of the machine. See the root build.gradle for the rationale. */ +final class ActiveProcessorCountArgumentProvider implements CommandLineArgumentProvider { + + @Input + final int processorCount + + ActiveProcessorCountArgumentProvider(int availableProcessorCount, int maxWorkerCount) { + // Floor of 2 keeps G1. -PmaxTestParallel=1 still sees 2 processors when + // maxWorkerCount is the usual unrestricted local worker pool. + this.processorCount = Math.max(2, (availableProcessorCount / Math.max(1, maxWorkerCount)) as int) + } + + @Override + Iterable asArguments() { + ["-XX:ActiveProcessorCount=$processorCount".toString()] + } +} + +subprojects { + tasks.withType(Test).configureEach { testTask -> + testTask.jvmArgumentProviders.add(new ActiveProcessorCountArgumentProvider( + Runtime.runtime.availableProcessors(), gradle.startParameter.maxWorkerCount)) + } } apply { diff --git a/grails-gradle/gradle/test-config.gradle b/grails-gradle/gradle/test-config.gradle index 8011f65a069..c6662764c80 100644 --- a/grails-gradle/gradle/test-config.gradle +++ b/grails-gradle/gradle/test-config.gradle @@ -59,7 +59,7 @@ tasks.withType(Test).configureEach { showStackTraces = true } excludes = ['**/*TestCase.class', '**/*$*.class'] - maxParallelForks = configuredTestParallel + maxParallelForks = rootProject.ext.configuredTestParallel maxHeapSize = isCiBuild ? '768m' : '1024m' if (System.getProperty('debug.tests')) { jvmArgs += debugArguments diff --git a/grails-gradle/plugins/src/test/groovy/org/grails/gradle/plugin/core/ActiveProcessorCountForkSpec.groovy b/grails-gradle/plugins/src/test/groovy/org/grails/gradle/plugin/core/ActiveProcessorCountForkSpec.groovy new file mode 100644 index 00000000000..051603402a7 --- /dev/null +++ b/grails-gradle/plugins/src/test/groovy/org/grails/gradle/plugin/core/ActiveProcessorCountForkSpec.groovy @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.grails.gradle.plugin.core + +import java.lang.management.ManagementFactory + +import spock.lang.Specification + +class ActiveProcessorCountForkSpec extends Specification { + + private static final String FLAG_PREFIX = '-XX:ActiveProcessorCount=' + + void 'forked test JVM receives ActiveProcessorCount from jvmArgumentProviders'() { + given: + String flag = ManagementFactory.runtimeMXBean.inputArguments.find { it.startsWith(FLAG_PREFIX) } + + expect: + flag != null + int advertised = Integer.parseInt(flag.substring(FLAG_PREFIX.length())) + advertised >= 2 + Runtime.runtime.availableProcessors() == advertised + } +} diff --git a/grails-test-suite-persistence/build.gradle b/grails-test-suite-persistence/build.gradle index 73173a79c56..476fd2609b1 100644 --- a/grails-test-suite-persistence/build.gradle +++ b/grails-test-suite-persistence/build.gradle @@ -88,7 +88,7 @@ dependencies { } test { - maxParallelForks = configuredTestParallel + maxParallelForks = rootProject.ext.configuredTestParallel forkEvery = isCiBuild ? 25 : 100 if(!isCiBuild) { maxHeapSize = '2048m' diff --git a/grails-test-suite-uber/build.gradle b/grails-test-suite-uber/build.gradle index 4fc49f517d5..74369b5e5bd 100644 --- a/grails-test-suite-uber/build.gradle +++ b/grails-test-suite-uber/build.gradle @@ -122,11 +122,6 @@ isolatedTestPatterns.keySet().each { taskName -> } } -tasks.named('isolatedTestsTwo', Test) { - maxParallelForks = 1 - forkEvery = 100 -} - tasks.withType(Test).configureEach { // Honor DO_NOT_CACHE_TESTS=1 so developers can repeatedly invoke the same test command // without --rerun-tasks (and without recompiling everything else). @@ -135,12 +130,20 @@ tasks.withType(Test).configureEach { onlyIf { !testSkippingProperties.any {project.hasProperty(it) } } useJUnitPlatform() - maxParallelForks = configuredTestParallel + maxParallelForks = rootProject.ext.configuredTestParallel forkEvery = isCiBuild ? 50 : 100 maxHeapSize = isCiBuild ? '768m' : '1024m' jvmArgs('--add-opens=java.base/java.lang=ALL-UNNAMED', '--add-opens=java.base/java.util=ALL-UNNAMED') } +// Must be configured AFTER the tasks.withType(Test) block above. Both are deferred +// configuration actions and Gradle runs them in registration order, so a tasks.named +// block registered first would be silently overwritten by the later configureEach. +tasks.named('isolatedTestsTwo', Test) { + maxParallelForks = 1 + forkEvery = 100 +} + tasks.named('test', Test) { // Exclude the isolated tests from the main test task filter.excludePatterns = isolatedTestPatterns.values().flatten() diff --git a/grails-test-suite-web/build.gradle b/grails-test-suite-web/build.gradle index 140465335c7..8281c284b53 100644 --- a/grails-test-suite-web/build.gradle +++ b/grails-test-suite-web/build.gradle @@ -53,7 +53,7 @@ dependencies { } def defaultTestConfig = { - maxParallelForks = configuredTestParallel + maxParallelForks = rootProject.ext.configuredTestParallel forkEvery = isCiBuild ? 50 : 100 excludes = ['**/*TestCase.class', '**/*$*.class'] }