From 5db34afaa0f0df6dc91b103c22cf63003d5be8f9 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 1 Jul 2026 18:42:39 -0400 Subject: [PATCH 01/36] DAOS-19247 test: Support scripted Test stages Add support for scripted fault injection and RPM test stages. Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 117 +++++++++++----------------- vars/scriptedTestStage.groovy | 118 +++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 73 deletions(-) create mode 100644 vars/scriptedTestStage.groovy diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 406cc0bbe..5dab561d3 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -61,81 +61,52 @@ Map call(Map kwargs = [:]) { Map job_status = kwargs.get('job_status', [:]) - return { - stage("${name}") { - // Get the tags for the stage. Use either the build parameter, commit pragma, or - // default tags. All tags are combined with the stage tags to ensure only tests that - // 'fit' the cluster will be run. - String tags = getFunctionalTags( - pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags) + String tags = getFunctionalTags( + pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags) - if (runStage == 'false') { - println("[${name}] Stage skipped by runStage=false") - Utils.markStageSkippedForConditional("${name}") - return - } else if (runStage == 'undefined') { - // To be removed once all stages have been converted to use runStage. - Map skip_kwargs = [ - 'tags': tags, - 'pragma_suffix': pragma_suffix, - 'distro': distro, - 'run_if_pr': run_if_pr, - 'run_if_landing': run_if_landing] - if (skipFunctionalTestStage(skip_kwargs)) { - println("[${name}] Stage skipped by skipFunctionalTestStage()") - Utils.markStageSkippedForConditional("${name}") - return - } - } else if (!testsInStage(tags)) { - println("[${name}] Stage skipped by no tests matching the '${tags}' tags") - Utils.markStageSkippedForConditional("${name}") - return - } + // Backwards compatibility: if runStage is not specified, use skipFunctionalTestStage() to + // determine if the stage should be run. + if (runStage == 'undefined') { + runStage = !skipFunctionalTestStage( + tags: tags, + pragma_suffix: pragma_suffix, + distro: distro, + run_if_pr: run_if_pr, + run_if_landing: run_if_landing).toString() + } - node(cachedCommitPragma("Test-label${pragma_suffix}", label)) { - // Ensure access to any branch provisioning scripts exist - println("[${name}] Check out '${base_branch}' from version control") - if (base_branch) { - checkoutScm( - url: 'https://github.com/daos-stack/daos.git', - branch: base_branch, - withSubmodules: false, - pruneStaleBranch: true) - } else { - checkoutScm(pruneStaleBranch: true) - } + // Avoid extra processing by skipping early if the stage is not to be run. + if (runStage == false) { + println("[${name}] Stage skipped by runStage=false") + Utils.markStageSkippedForConditional("${name}") + return + } - try { - println("[${name}] Running functionalTest() on ${label} with tags=${tags}") - Map ftestConfig = [ - image_version: image_version, - inst_repos: daosRepos(distro), - inst_rpms: functionalPackages( - clientVersion: 1, - nextVersion: next_version, - addDaosPkgs: 'tests-internal', - rpmDistribution: rpm_distro) + ' ' + other_packages, - test_tag: tags, - ftest_arg: getFunctionalArgs( - pragma_suffix: pragma_suffix, - nvme: nvme, - default_nvme: default_nvme, - provider: provider)['ftest_arg'], - test_function: 'runTestFunctionalV2'] - if (node_count != null) { - ftestConfig['node_count'] = node_count - } - jobStatusUpdate( - job_status, - name, - functionalTest(ftestConfig)) - } finally { - println("[${name}] Running functionalTestPostV2()") - functionalTestPostV2() - jobStatusUpdate(job_status, name) - } - } - } - println("[${name}] Finished with ${job_status}") + Map functionalTestArgs = [ + image_version: image_version, + inst_repos: daosRepos(distro), + inst_rpms: functionalPackages( + clientVersion: 1, + nextVersion: next_version, + addDaosPkgs: 'tests-internal', + rpmDistribution: rpm_distro) + ' ' + other_packages, + test_tag: tags, + ftest_arg: getFunctionalArgs( + pragma_suffix: pragma_suffix, + nvme: nvme, + default_nvme: default_nvme, + provider: provider)['ftest_arg'], + test_function: 'runTestFunctionalV2'] + if (node_count != null) { + functionalTestArgs['node_count'] = node_count } + + return scriptedTestStage( + name: name, + pragmaSuffix: pragma_suffix, + label: label, + testBranch: base_branch, + jobStatus: job_status, + functionalTestArgs: functionalTestArgs, + ) } diff --git a/vars/scriptedTestStage.groovy b/vars/scriptedTestStage.groovy new file mode 100644 index 000000000..ae38ddaf1 --- /dev/null +++ b/vars/scriptedTestStage.groovy @@ -0,0 +1,118 @@ +// vars/scriptedTestStage.groovy + +import org.jenkinsci.plugins.pipeline.modeldefinition.Utils + +/** + * scriptedTestStage.groovy + * + * Get a test stage in scripted syntax. + * + * @param kwargs Map containing the following optional arguments (empty strings yield defaults): + * name test stage name + * runStage whether or not to run the test stage + * pragmaSuffix test stage commit pragma suffix, e.g. '-hw-medium' + * label test stage default cluster label + * testBranch if specified, checkout sources from this branch before running tests + * jobStatus Map of status for each stage in the job/build + * functionalTestArgs Map of arguments to pass to functionalTest() for the stage + * unitTestArgs Map of arguments to pass to unitTest() for the stage + * unitTestPostArgs Map of arguments to pass to unitTestPost() for the stage + * testRpmArgs Map of arguments to pass to testRpm() for the stage + * testRpmPostArgs Map of arguments to pass to testRpmPost() for the stage + * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage + * @return a scripted stage to run in a pipeline + */ +Map call(Map kwargs = [:]) { + // General parameters + String name = kwargs.get('name', 'Unknown Functional Test Stage') + Boolean runStage = kwargs.get('runStage', true) as Boolean + String pragmaSuffix = kwargs.get('pragmaSuffix', '') + String label = kwargs.get('label') + String testBranch = kwargs.get('testBranch', '') + Map jobStatus = kwargs.get('jobStatus', [:]) + + // Functional test stage parameters + Map functionalTestArgs = kwargs.get('functionalTestArgs', [:]) + + // Unit Test stage parameters + Map unitTestArgs = kwargs.get('unitTestArgs', [:]) + Map unitTestPostArgs = kwargs.get('unitTestPostArgs', [:]) + + // Test RPM stage parameters + Map testRpmArgs = kwargs.get('testRpmArgs', [:]) + Map testRpmPostArgs = kwargs.get('testRpmPostArgs', [:]) + + // Artifact archiving stage parameters + Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', [:]) + + return { + stage("${name}") { + if (!runStage) { + println("[${name}] Stage skipped by runStage=false") + Utils.markStageSkippedForConditional("${name}") + return + } + + if (pragmaSuffix) { + label = cachedCommitPragma("Test-label${pragmaSuffix}", label) + } + + node(label) { + // Ensure access to any branch provisioning scripts exist + if (testBranch) { + println("[${name}] Check out '${testBranch}' from version control") + checkoutScm( + url: 'https://github.com/daos-stack/daos.git', + branch: testBranch, + withSubmodules: false, + pruneStaleBranch: true) + } else { + println("[${name}] Check out branch from version control") + checkoutScm(pruneStaleBranch: true) + } + + try { + if (functionalTestArgs) { + println("[${name}] Running functionalTest() on ${label} with tags=${tags}") + jobStatusUpdate(jobStatus, name, functionalTest(functionalTestArgs)) + } else if (unitTestArgs) { + println("[${name}] Running unitTest() on ${label} with tags=${tags}") + jobStatusUpdate(jobStatus, name, unitTest(unitTestArgs)) + } else if (testRpmArgs) { + println("[${name}] Running testRpm() on ${label} with tags=${tags}") + jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs)) + } else { + println("[${name}] No test arguments provided!") + } + } finally { + if (functionalTestArgs) { + println("[${name}] Running functionalTestPostV2()") + functionalTestPostV2() + } + if (unitTestPostArgs) { + println("[${name}] Running unitTestPost()") + unitTestPost(unitTestPostArgs) + } + if (archiveArtifactsArgs) { + println("[${name}] Running archiveArtifacts()") + archiveArtifacts(archiveArtifactsArgs) + } + if (testRpmArgs) { + println("[${name}] Running archiveArtifacts()") + // Extract first node from comma-delimited list + String firstNode = env.NODELIST.split(',')[0].trim() + sh label: 'Fetch and stage artifacts', + script: "hostname; ssh -i ci_key jenkins@ ${firstNode}" + + " ls -ltar /tmp; mkdir -p \"${name}\" && " + + "scp -i ci_key jenkins@${firstNode}" + + ':/tmp/{{suite_dmg,daos_{server_helper,{control,agent}}}.log,daos_server.log.*}' + + " \"${name}/\"" + archiveArtifacts(artifacts: "${name}/**") + } + jobStatusUpdate(jobStatus, name) + } + } + println("[${name}] Finished with ${jobStatus}") + } + } +} From ef5d151f3fa09dbad9ad2585da4f770f387d9bec Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 1 Jul 2026 22:53:14 -0400 Subject: [PATCH 02/36] Restore getFunctionalTestStage.groovy Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 117 ++++++++++++++++++----------- vars/scriptedTestStage.groovy | 6 ++ 2 files changed, 79 insertions(+), 44 deletions(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 5dab561d3..406cc0bbe 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -61,52 +61,81 @@ Map call(Map kwargs = [:]) { Map job_status = kwargs.get('job_status', [:]) - String tags = getFunctionalTags( - pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags) + return { + stage("${name}") { + // Get the tags for the stage. Use either the build parameter, commit pragma, or + // default tags. All tags are combined with the stage tags to ensure only tests that + // 'fit' the cluster will be run. + String tags = getFunctionalTags( + pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags) - // Backwards compatibility: if runStage is not specified, use skipFunctionalTestStage() to - // determine if the stage should be run. - if (runStage == 'undefined') { - runStage = !skipFunctionalTestStage( - tags: tags, - pragma_suffix: pragma_suffix, - distro: distro, - run_if_pr: run_if_pr, - run_if_landing: run_if_landing).toString() - } + if (runStage == 'false') { + println("[${name}] Stage skipped by runStage=false") + Utils.markStageSkippedForConditional("${name}") + return + } else if (runStage == 'undefined') { + // To be removed once all stages have been converted to use runStage. + Map skip_kwargs = [ + 'tags': tags, + 'pragma_suffix': pragma_suffix, + 'distro': distro, + 'run_if_pr': run_if_pr, + 'run_if_landing': run_if_landing] + if (skipFunctionalTestStage(skip_kwargs)) { + println("[${name}] Stage skipped by skipFunctionalTestStage()") + Utils.markStageSkippedForConditional("${name}") + return + } + } else if (!testsInStage(tags)) { + println("[${name}] Stage skipped by no tests matching the '${tags}' tags") + Utils.markStageSkippedForConditional("${name}") + return + } - // Avoid extra processing by skipping early if the stage is not to be run. - if (runStage == false) { - println("[${name}] Stage skipped by runStage=false") - Utils.markStageSkippedForConditional("${name}") - return - } + node(cachedCommitPragma("Test-label${pragma_suffix}", label)) { + // Ensure access to any branch provisioning scripts exist + println("[${name}] Check out '${base_branch}' from version control") + if (base_branch) { + checkoutScm( + url: 'https://github.com/daos-stack/daos.git', + branch: base_branch, + withSubmodules: false, + pruneStaleBranch: true) + } else { + checkoutScm(pruneStaleBranch: true) + } - Map functionalTestArgs = [ - image_version: image_version, - inst_repos: daosRepos(distro), - inst_rpms: functionalPackages( - clientVersion: 1, - nextVersion: next_version, - addDaosPkgs: 'tests-internal', - rpmDistribution: rpm_distro) + ' ' + other_packages, - test_tag: tags, - ftest_arg: getFunctionalArgs( - pragma_suffix: pragma_suffix, - nvme: nvme, - default_nvme: default_nvme, - provider: provider)['ftest_arg'], - test_function: 'runTestFunctionalV2'] - if (node_count != null) { - functionalTestArgs['node_count'] = node_count + try { + println("[${name}] Running functionalTest() on ${label} with tags=${tags}") + Map ftestConfig = [ + image_version: image_version, + inst_repos: daosRepos(distro), + inst_rpms: functionalPackages( + clientVersion: 1, + nextVersion: next_version, + addDaosPkgs: 'tests-internal', + rpmDistribution: rpm_distro) + ' ' + other_packages, + test_tag: tags, + ftest_arg: getFunctionalArgs( + pragma_suffix: pragma_suffix, + nvme: nvme, + default_nvme: default_nvme, + provider: provider)['ftest_arg'], + test_function: 'runTestFunctionalV2'] + if (node_count != null) { + ftestConfig['node_count'] = node_count + } + jobStatusUpdate( + job_status, + name, + functionalTest(ftestConfig)) + } finally { + println("[${name}] Running functionalTestPostV2()") + functionalTestPostV2() + jobStatusUpdate(job_status, name) + } + } + } + println("[${name}] Finished with ${job_status}") } - - return scriptedTestStage( - name: name, - pragmaSuffix: pragma_suffix, - label: label, - testBranch: base_branch, - jobStatus: job_status, - functionalTestArgs: functionalTestArgs, - ) } diff --git a/vars/scriptedTestStage.groovy b/vars/scriptedTestStage.groovy index ae38ddaf1..faa4f3fbc 100644 --- a/vars/scriptedTestStage.groovy +++ b/vars/scriptedTestStage.groovy @@ -47,6 +47,8 @@ Map call(Map kwargs = [:]) { return { stage("${name}") { + println("[${name}] Starting stage: kwargs=${kwargs}") + if (!runStage) { println("[${name}] Stage skipped by runStage=false") Utils.markStageSkippedForConditional("${name}") @@ -58,6 +60,7 @@ Map call(Map kwargs = [:]) { } node(label) { + println("[${name}] DEBUG: node(${label}) started") // Ensure access to any branch provisioning scripts exist if (testBranch) { println("[${name}] Check out '${testBranch}' from version control") @@ -72,6 +75,7 @@ Map call(Map kwargs = [:]) { } try { + println("[${name}] DEBUG: try{} started") if (functionalTestArgs) { println("[${name}] Running functionalTest() on ${label} with tags=${tags}") jobStatusUpdate(jobStatus, name, functionalTest(functionalTestArgs)) @@ -83,8 +87,10 @@ Map call(Map kwargs = [:]) { jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs)) } else { println("[${name}] No test arguments provided!") + jobStatusUpdate(jobStatus, name, 'FAILED') } } finally { + println("[${name}] DEBUG: finally{} started") if (functionalTestArgs) { println("[${name}] Running functionalTestPostV2()") functionalTestPostV2() From 09d02efa12e0ea934120c6b35c558bda6c66061f Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 1 Jul 2026 23:35:17 -0400 Subject: [PATCH 03/36] Updates. Signed-off-by: Phil Henderson --- vars/scriptedTestStage.groovy | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/vars/scriptedTestStage.groovy b/vars/scriptedTestStage.groovy index faa4f3fbc..1da517e14 100644 --- a/vars/scriptedTestStage.groovy +++ b/vars/scriptedTestStage.groovy @@ -79,15 +79,14 @@ Map call(Map kwargs = [:]) { if (functionalTestArgs) { println("[${name}] Running functionalTest() on ${label} with tags=${tags}") jobStatusUpdate(jobStatus, name, functionalTest(functionalTestArgs)) - } else if (unitTestArgs) { + } + if (unitTestArgs) { println("[${name}] Running unitTest() on ${label} with tags=${tags}") jobStatusUpdate(jobStatus, name, unitTest(unitTestArgs)) - } else if (testRpmArgs) { + } + if (testRpmArgs) { println("[${name}] Running testRpm() on ${label} with tags=${tags}") jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs)) - } else { - println("[${name}] No test arguments provided!") - jobStatusUpdate(jobStatus, name, 'FAILED') } } finally { println("[${name}] DEBUG: finally{} started") From 0f14bd50fbe6768fda79d119b0e09c0a8a6f49e3 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 1 Jul 2026 23:59:52 -0400 Subject: [PATCH 04/36] Debug Signed-off-by: Phil Henderson --- vars/scriptedTestStage.groovy | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/vars/scriptedTestStage.groovy b/vars/scriptedTestStage.groovy index 1da517e14..75760b45b 100644 --- a/vars/scriptedTestStage.groovy +++ b/vars/scriptedTestStage.groovy @@ -48,6 +48,11 @@ Map call(Map kwargs = [:]) { return { stage("${name}") { println("[${name}] Starting stage: kwargs=${kwargs}") + println("[${name}] Debug: unitTestArgs: empty=${unitTestArgs.isEmpty()}; ${unitTestArgs}") + println("[${name}] Debug: unitTestPostArgs: empty=${unitTestPostArgs.isEmpty()}; ${unitTestPostArgs}") + println("[${name}] Debug: testRpmArgs: empty=${testRpmArgs.isEmpty()}; ${testRpmArgs}") + println("[${name}] Debug: testRpmPostArgs: empty=${testRpmPostArgs.isEmpty()}; ${testRpmPostArgs}") + println("[${name}] Debug: archiveArtifactsArgs: empty=${archiveArtifactsArgs.isEmpty()}; ${archiveArtifactsArgs}") if (!runStage) { println("[${name}] Stage skipped by runStage=false") @@ -75,7 +80,7 @@ Map call(Map kwargs = [:]) { } try { - println("[${name}] DEBUG: try{} started") + println("[${name}] Running test steps on ${label}") if (functionalTestArgs) { println("[${name}] Running functionalTest() on ${label} with tags=${tags}") jobStatusUpdate(jobStatus, name, functionalTest(functionalTestArgs)) @@ -89,7 +94,7 @@ Map call(Map kwargs = [:]) { jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs)) } } finally { - println("[${name}] DEBUG: finally{} started") + println("[${name}] Running test cleanup on ${label}") if (functionalTestArgs) { println("[${name}] Running functionalTestPostV2()") functionalTestPostV2() From d677368fd73b667d7fbd471afcc8432ada47abeb Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 00:20:34 -0400 Subject: [PATCH 05/36] Debug Signed-off-by: Phil Henderson --- vars/scriptedTestStage.groovy | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/vars/scriptedTestStage.groovy b/vars/scriptedTestStage.groovy index 75760b45b..6504ba504 100644 --- a/vars/scriptedTestStage.groovy +++ b/vars/scriptedTestStage.groovy @@ -29,25 +29,25 @@ Map call(Map kwargs = [:]) { String pragmaSuffix = kwargs.get('pragmaSuffix', '') String label = kwargs.get('label') String testBranch = kwargs.get('testBranch', '') - Map jobStatus = kwargs.get('jobStatus', [:]) + Map jobStatus = kwargs.get('jobStatus', null) ?: [:] // Functional test stage parameters - Map functionalTestArgs = kwargs.get('functionalTestArgs', [:]) + Map functionalTestArgs = kwargs.get('functionalTestArgs', null) ?: [:] // Unit Test stage parameters - Map unitTestArgs = kwargs.get('unitTestArgs', [:]) - Map unitTestPostArgs = kwargs.get('unitTestPostArgs', [:]) + Map unitTestArgs = kwargs.get('unitTestArgs', null) ?: [:] + Map unitTestPostArgs = kwargs.get('unitTestPostArgs', null) ?: [:] // Test RPM stage parameters - Map testRpmArgs = kwargs.get('testRpmArgs', [:]) - Map testRpmPostArgs = kwargs.get('testRpmPostArgs', [:]) + Map testRpmArgs = kwargs.get('testRpmArgs', null) ?: [:] + Map testRpmPostArgs = kwargs.get('testRpmPostArgs', null) ?: [:] // Artifact archiving stage parameters - Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', [:]) + Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:] return { stage("${name}") { - println("[${name}] Starting stage: kwargs=${kwargs}") + println("[${name}] Starting stage: kwargs keys=${kwargs.keySet()}") println("[${name}] Debug: unitTestArgs: empty=${unitTestArgs.isEmpty()}; ${unitTestArgs}") println("[${name}] Debug: unitTestPostArgs: empty=${unitTestPostArgs.isEmpty()}; ${unitTestPostArgs}") println("[${name}] Debug: testRpmArgs: empty=${testRpmArgs.isEmpty()}; ${testRpmArgs}") @@ -81,33 +81,37 @@ Map call(Map kwargs = [:]) { try { println("[${name}] Running test steps on ${label}") - if (functionalTestArgs) { + if (!functionalTestArgs.isEmpty()) { println("[${name}] Running functionalTest() on ${label} with tags=${tags}") jobStatusUpdate(jobStatus, name, functionalTest(functionalTestArgs)) } - if (unitTestArgs) { + if (!unitTestArgs.isEmpty()) { println("[${name}] Running unitTest() on ${label} with tags=${tags}") jobStatusUpdate(jobStatus, name, unitTest(unitTestArgs)) } - if (testRpmArgs) { + if (!testRpmArgs.isEmpty()) { println("[${name}] Running testRpm() on ${label} with tags=${tags}") jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs)) } + } catch (Exception e) { + println("[${name}] Caught exception: ${e}") + jobStatusUpdate(jobStatus, name, 'FAILURE') + throw e } finally { println("[${name}] Running test cleanup on ${label}") - if (functionalTestArgs) { + if (!functionalTestArgs.isEmpty()) { println("[${name}] Running functionalTestPostV2()") functionalTestPostV2() } - if (unitTestPostArgs) { + if (!unitTestPostArgs.isEmpty()) { println("[${name}] Running unitTestPost()") unitTestPost(unitTestPostArgs) } - if (archiveArtifactsArgs) { + if (!archiveArtifactsArgs.isEmpty()) { println("[${name}] Running archiveArtifacts()") archiveArtifacts(archiveArtifactsArgs) } - if (testRpmArgs) { + if (!testRpmArgs.isEmpty()) { println("[${name}] Running archiveArtifacts()") // Extract first node from comma-delimited list String firstNode = env.NODELIST.split(',')[0].trim() From 82667f5447b62aea752d671b484029eb04444b84 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 01:24:10 -0400 Subject: [PATCH 06/36] Updates Signed-off-by: Phil Henderson --- vars/scriptedDockerStage.groovy | 133 ++++++++++++++++++++++++++++++ vars/scriptedTestRpmStage.groovy | 88 ++++++++++++++++++++ vars/scriptedTestStage.groovy | 132 ----------------------------- vars/scriptedUnitTestStage.groovy | 100 ++++++++++++++++++++++ 4 files changed, 321 insertions(+), 132 deletions(-) create mode 100644 vars/scriptedDockerStage.groovy create mode 100644 vars/scriptedTestRpmStage.groovy delete mode 100644 vars/scriptedTestStage.groovy create mode 100644 vars/scriptedUnitTestStage.groovy diff --git a/vars/scriptedDockerStage.groovy b/vars/scriptedDockerStage.groovy new file mode 100644 index 000000000..9921c8301 --- /dev/null +++ b/vars/scriptedDockerStage.groovy @@ -0,0 +1,133 @@ +// vars/scriptedDockerStage.groovy + +import org.jenkinsci.plugins.pipeline.modeldefinition.Utils + +/** + * scriptedDockerStage + * + * Get a build stage in scripted syntax. + * + * @param kwargs Map containing the following optional arguments (empty strings yield defaults): + * name the build stage name + * runStage optional additional condition to determine if the stage runs + * jobStatus Map of status for each stage in the job/build + * dockerTag the docker image tag to use for the build + * dockerBuildArgs optional docker build arguments + * buildRpms whether or not to build rpms; defaults to true + * distro the shorthand distro name; defaults to 'el8' + * rpmDistro the distro to use for rpm building; defaults to distro + * release the DAOS RPM release value to use; defaults to env.DAOS_RELVAL + * compiler the compiler to use; defaults to 'gcc' + * sconsBuildArgs optional scons build arguments + * valgrindSconsBuildArgs optional scons build arguments for valgrind build + * artifacts optional artifacts name to archive; defaults to + * "config.log-${distro}-${compiler}" + * uploadTarget the distro to use when uploading rpms; defaults to distro + * installScript optional script to run to install rpms; defaults to '' + * publishHtmlArgs optional arguments to pass to publishHTML() + * archiveArtifactsArgs optional arguments to pass to archiveArtifacts() + * @return a scripted stage to run in a pipeline + */ +Map call(Map kwargs = [:]) { + String name = kwargs.get('name', 'Unknown Docker Stage') + Boolean runStage = kwargs.get('runStage', true) + Map jobStatus = kwargs.get('jobStatus', [:]) + String dockerTag = jobStatusKey("build-${uploadTarget}-${compiler}").toLowerCase() + String dockerBuildArgs = kwargs.get('dockerBuildArgs', '') + + // Build stage parameters + Boolean buildRpms = kwargs.get('buildRpms', true) + String distro = kwargs.get('distro', 'el8') + String rpmDistro = kwargs.get('rpmDistro', distro) + String release = kwargs.get('release', env.DAOS_RELVAL) + String compiler = kwargs.get('compiler', 'gcc') + Map sconsBuildArgs = kwargs.get('sconsBuildArgs', [:]) + Map valgrindSconsBuildArgs = kwargs.get('valgrindSconsBuildArgs', [:]) + String artifacts = kwargs.get('artifacts', "config.log-${distro}-${compiler}") + String uploadTarget = kwargs.get('uploadTarget', distro) + String bullseye = 'false' + if (compiler == 'covc') { + bullseye = 'true' + } + + // Summary stage parameters + String installScript = kwargs.get('installScript', '') + Map publishHtmlArgs = kwargs.get('publishHtmlArgs', [:]) + Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', [:]) + + return { + stage("${name}") { + if (!runStage) { + println("[${name}] Marking docker stage as skipped") + Utils.markStageSkippedForConditional("${name}") + return + } + node('docker_runner') { + println("[${name}] Check out from version control") + checkoutScm(pruneStaleBranch: true) + + def dockerImage = docker.build(dockerTag, dockerBuildArgs) + try { + dockerImage.inside() { + if (buildRpms) { + sh label: 'Install RPMs', + script: "./ci/rpm/install_deps.sh ${rpmDistro} ${release} ${bullseye}" + // Avoid interpolation of sensitive environment variables + sh label: 'Build deps', + script: "./ci/rpm/build_deps.sh ${bullseye}" + ' ${BULLSEYE_KEY}' + } + if (sconsBuildArgs) { + job_step_update(sconsBuild(sconsBuildArgs)) + } + if (valgrindSconsBuildArgs) { + // For non-release builds, create a separate build with the valgrind + // tag for NLT memcheck testing. This is necessary to avoid problems + // caused by valgrind being confused by the Go runtime. We don't want + // to use the valgrind build for normal testing because it is much + // slower. BUILD_TYPE=dev is set for PR/dev builds in sconsArgs(), and + // TARGET_TYPE=release is used to select pre-built cached prerequisites. + job_step_update(sconsBuild(valgrindSconsBuildArgs)) + sh label: 'Stash valgrind install tree for NLT', + script: 'tar -C / -cf opt-daos-valgrind.tar opt/daos' + stash(name: 'opt-daos-valgrind', includes: 'opt-daos-valgrind.tar') + } + if (buildRpms) { + sh label: 'Generate RPMs', + script: "./ci/rpm/gen_rpms.sh ${rpmDistro} ${release} ${bullseye}" + // Success actions + uploadNewRPMs(uploadTarget, 'success') + } + if (installScript) { + sh label: 'Install RPMs', + script: "${installScript} ${distro}" + } + } + } catch (Exception e) { + println("[${name}] Caught exception: ${e}") + if (sconsBuildArgs) { + // Unsuccessful actions + sh """if [ -f config.log ]; then + mv config.log ${artifacts} + fi""" + archiveArtifacts artifacts: "${artifacts}", allowEmptyArchive: true + } + jobStatusUpdate(jobStatus, name, 'FAILURE') + throw e + } finally { + // Cleanup actions + if (buildRpms) { + uploadNewRPMs(uploadTarget, 'cleanup') + } + if (publishHtmlArgs) { + publishHTML(publishHtmlArgs) + } + if (archiveArtifactsArgs) { + archiveArtifacts(archiveArtifactsArgs) + } + jobStatusUpdate(jobStatus, name) + } + } + println("[${name}] Finished with ${jobStatus}") + } + } +} diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy new file mode 100644 index 000000000..b67e2174e --- /dev/null +++ b/vars/scriptedTestRpmStage.groovy @@ -0,0 +1,88 @@ +// vars/scriptedTestRpmStage.groovy + +import org.jenkinsci.plugins.pipeline.modeldefinition.Utils + +/** + * scriptedTestRpmStage.groovy + * + * Get a test stage in scripted syntax. + * + * @param kwargs Map containing the following optional arguments (empty strings yield defaults): + * name test stage name + * runStage whether or not to run the test stage + * label test stage default cluster label + * testBranch if specified, checkout sources from this branch before running tests + * jobStatus Map of status for each stage in the job/build + * instRepos testRpm() inst_repos argument; defaults to daosRepos() + * daosPkgVersion testRpm() daos_pkg_version argument; defaults to daosPackagesVersion(next_version()) + * instRpms testRpm() inst_rpms argument; defaults to 'mercury-libfabric' + * ignoreFailure testRpm() ignore_failure argument; defaults to false + * nodeList comma-delimited list of nodes; defaults to env.NODELIST + * @return a scripted stage to run in a pipeline + */ +Map call(Map kwargs = [:]) { + // General parameters + String name = kwargs.get('name', 'Unknown Functional Test Stage') + Boolean runStage = kwargs.get('runStage', true) as Boolean + String label = kwargs.get('label') + String testBranch = kwargs.get('testBranch', '') + Map jobStatus = kwargs.get('jobStatus', null) ?: [:] + Map testRpmArgs = [ + inst_repos: kwargs.get('instRepos', daosRepos()), + daos_pkg_version: kwargs.get( + 'daosPkgVersion', daosPackagesVersion(kwargs.get('next_version', null))), + inst_rpms: kwargs.get('instRpms', 'mercury-libfabric'), + ignore_failure: kwargs.get('ignoreFailure', false) + ] + String nodeList = kwargs.get('nodeList', env.NODELIST) + + return { + stage("${name}") { + println("[${name}] Starting stage: kwargs keys=${kwargs.keySet()}") + + if (!runStage) { + println("[${name}] Stage skipped by runStage=false") + Utils.markStageSkippedForConditional("${name}") + return + } + + node(label) { + // Ensure access to any branch provisioning scripts exist + if (testBranch) { + println("[${name}] Check out '${testBranch}' from version control") + checkoutScm( + url: 'https://github.com/daos-stack/daos.git', + branch: testBranch, + withSubmodules: false, + pruneStaleBranch: true) + } else { + println("[${name}] Check out branch from version control") + checkoutScm(pruneStaleBranch: true) + } + + try { + println("[${name}] Running testRpm() on ${label}") + jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs)) + } catch (Exception e) { + println("[${name}] Caught exception: ${e}") + jobStatusUpdate(jobStatus, name, 'FAILURE') + throw e + } finally { + println("[${name}] Running archiveArtifacts()") + // Extract first node from comma-delimited list + String firstNode = nodeList.split(',')[0].trim() + sh label: 'Fetch and stage artifacts', + script: "hostname; ssh -i ci_key jenkins@${firstNode}" + + " ls -ltar /tmp; mkdir -p \"${name}\" && " + + "scp -i ci_key jenkins@${firstNode}:/tmp/" + + '{{suite_dmg,daos_{server_helper,{control,agent}}}.log,' + + 'daos_server.log.*}' + + " \"${name}/\"" + archiveArtifacts(artifacts: "${name}/**") + jobStatusUpdate(jobStatus, name) + } + } + println("[${name}] Finished with ${jobStatus}") + } + } +} diff --git a/vars/scriptedTestStage.groovy b/vars/scriptedTestStage.groovy deleted file mode 100644 index 6504ba504..000000000 --- a/vars/scriptedTestStage.groovy +++ /dev/null @@ -1,132 +0,0 @@ -// vars/scriptedTestStage.groovy - -import org.jenkinsci.plugins.pipeline.modeldefinition.Utils - -/** - * scriptedTestStage.groovy - * - * Get a test stage in scripted syntax. - * - * @param kwargs Map containing the following optional arguments (empty strings yield defaults): - * name test stage name - * runStage whether or not to run the test stage - * pragmaSuffix test stage commit pragma suffix, e.g. '-hw-medium' - * label test stage default cluster label - * testBranch if specified, checkout sources from this branch before running tests - * jobStatus Map of status for each stage in the job/build - * functionalTestArgs Map of arguments to pass to functionalTest() for the stage - * unitTestArgs Map of arguments to pass to unitTest() for the stage - * unitTestPostArgs Map of arguments to pass to unitTestPost() for the stage - * testRpmArgs Map of arguments to pass to testRpm() for the stage - * testRpmPostArgs Map of arguments to pass to testRpmPost() for the stage - * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage - * @return a scripted stage to run in a pipeline - */ -Map call(Map kwargs = [:]) { - // General parameters - String name = kwargs.get('name', 'Unknown Functional Test Stage') - Boolean runStage = kwargs.get('runStage', true) as Boolean - String pragmaSuffix = kwargs.get('pragmaSuffix', '') - String label = kwargs.get('label') - String testBranch = kwargs.get('testBranch', '') - Map jobStatus = kwargs.get('jobStatus', null) ?: [:] - - // Functional test stage parameters - Map functionalTestArgs = kwargs.get('functionalTestArgs', null) ?: [:] - - // Unit Test stage parameters - Map unitTestArgs = kwargs.get('unitTestArgs', null) ?: [:] - Map unitTestPostArgs = kwargs.get('unitTestPostArgs', null) ?: [:] - - // Test RPM stage parameters - Map testRpmArgs = kwargs.get('testRpmArgs', null) ?: [:] - Map testRpmPostArgs = kwargs.get('testRpmPostArgs', null) ?: [:] - - // Artifact archiving stage parameters - Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:] - - return { - stage("${name}") { - println("[${name}] Starting stage: kwargs keys=${kwargs.keySet()}") - println("[${name}] Debug: unitTestArgs: empty=${unitTestArgs.isEmpty()}; ${unitTestArgs}") - println("[${name}] Debug: unitTestPostArgs: empty=${unitTestPostArgs.isEmpty()}; ${unitTestPostArgs}") - println("[${name}] Debug: testRpmArgs: empty=${testRpmArgs.isEmpty()}; ${testRpmArgs}") - println("[${name}] Debug: testRpmPostArgs: empty=${testRpmPostArgs.isEmpty()}; ${testRpmPostArgs}") - println("[${name}] Debug: archiveArtifactsArgs: empty=${archiveArtifactsArgs.isEmpty()}; ${archiveArtifactsArgs}") - - if (!runStage) { - println("[${name}] Stage skipped by runStage=false") - Utils.markStageSkippedForConditional("${name}") - return - } - - if (pragmaSuffix) { - label = cachedCommitPragma("Test-label${pragmaSuffix}", label) - } - - node(label) { - println("[${name}] DEBUG: node(${label}) started") - // Ensure access to any branch provisioning scripts exist - if (testBranch) { - println("[${name}] Check out '${testBranch}' from version control") - checkoutScm( - url: 'https://github.com/daos-stack/daos.git', - branch: testBranch, - withSubmodules: false, - pruneStaleBranch: true) - } else { - println("[${name}] Check out branch from version control") - checkoutScm(pruneStaleBranch: true) - } - - try { - println("[${name}] Running test steps on ${label}") - if (!functionalTestArgs.isEmpty()) { - println("[${name}] Running functionalTest() on ${label} with tags=${tags}") - jobStatusUpdate(jobStatus, name, functionalTest(functionalTestArgs)) - } - if (!unitTestArgs.isEmpty()) { - println("[${name}] Running unitTest() on ${label} with tags=${tags}") - jobStatusUpdate(jobStatus, name, unitTest(unitTestArgs)) - } - if (!testRpmArgs.isEmpty()) { - println("[${name}] Running testRpm() on ${label} with tags=${tags}") - jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs)) - } - } catch (Exception e) { - println("[${name}] Caught exception: ${e}") - jobStatusUpdate(jobStatus, name, 'FAILURE') - throw e - } finally { - println("[${name}] Running test cleanup on ${label}") - if (!functionalTestArgs.isEmpty()) { - println("[${name}] Running functionalTestPostV2()") - functionalTestPostV2() - } - if (!unitTestPostArgs.isEmpty()) { - println("[${name}] Running unitTestPost()") - unitTestPost(unitTestPostArgs) - } - if (!archiveArtifactsArgs.isEmpty()) { - println("[${name}] Running archiveArtifacts()") - archiveArtifacts(archiveArtifactsArgs) - } - if (!testRpmArgs.isEmpty()) { - println("[${name}] Running archiveArtifacts()") - // Extract first node from comma-delimited list - String firstNode = env.NODELIST.split(',')[0].trim() - sh label: 'Fetch and stage artifacts', - script: "hostname; ssh -i ci_key jenkins@ ${firstNode}" + - " ls -ltar /tmp; mkdir -p \"${name}\" && " + - "scp -i ci_key jenkins@${firstNode}" + - ':/tmp/{{suite_dmg,daos_{server_helper,{control,agent}}}.log,daos_server.log.*}' + - " \"${name}/\"" - archiveArtifacts(artifacts: "${name}/**") - } - jobStatusUpdate(jobStatus, name) - } - } - println("[${name}] Finished with ${jobStatus}") - } - } -} diff --git a/vars/scriptedUnitTestStage.groovy b/vars/scriptedUnitTestStage.groovy new file mode 100644 index 000000000..edc2c2de1 --- /dev/null +++ b/vars/scriptedUnitTestStage.groovy @@ -0,0 +1,100 @@ +// vars/scriptedUnitTestStage.groovy + +import org.jenkinsci.plugins.pipeline.modeldefinition.Utils + +/** + * scriptedUnitTestStage.groovy + * + * Get a unit test stage in scripted syntax. + * + * @param kwargs Map containing the following optional arguments (empty strings yield defaults): + * name test stage name + * runStage whether or not to run the test stage + * label test stage default cluster label + * testBranch if specified, checkout sources from this branch before running tests + * jobStatus Map of status for each stage in the job/build + * target the target distro to use for the test stage; defaults to 'el9' + * timeoutTime unitTest() timeout time in minutes; defaults to 60 + * instRepos unitTest() inst_repos argument; defaults to daosRepos() + * testScript unitTest() test_script argument; defaults to '' + * withValgrind unitTest() with_valgrind argument; defaults to '' + * alwaysScript unitTest() always_script argument; defaults to 'ci/unit/test_nlt_post.sh' + * testResults unitTest() test_results argument; defaults to 'nlt-junit.xml' + * unstashOpt unitTest() unstash_opt argument; defaults to true + * unstashTests unitTest() unstash_tests argument; defaults to false + * instRpms unitTest() inst_rpms argument; defaults to unitPackages(target: target) + ' daos-client-tests' + * imageVersion unitTest() image_version argument; defaults to 'el9.7' + * provEnvVars unitTest() prov_env_vars argument; defaults to '' + * unitTestPostArgs Map of arguments to pass to unitTestPost() for the stage + * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage + * @return a scripted stage to run in a pipeline + */ +Map call(Map kwargs = [:]) { + // General parameters + String name = kwargs.get('name', 'Unknown Functional Test Stage') + Boolean runStage = kwargs.get('runStage', true) as Boolean + String label = kwargs.get('label') + String testBranch = kwargs.get('testBranch', '') + Map jobStatus = kwargs.get('jobStatus', null) ?: [:] + String distro = kwargs.get('distro', 'el9') + + // Unit Test stage parameters + Map unitTestArgs = [ + 'timeout_time': kwargs.get('timeoutTime', 60), + 'inst_repos': kwargs.get('instRepos', daosRepos(distro)), + 'test_script': kwargs.get('testScript', ''), + 'with_valgrind': kwargs.get('withValgrind', ''), + 'always_script': kwargs.get('alwaysScript', 'ci/unit/test_nlt_post.sh'), + 'test_results': kwargs.get('testResults', 'nlt-junit.xml'), + 'unstash_opt': kwargs.get('unstashOpt', true), + 'unstash_tests': kwargs.get('unstashTests', false), + 'inst_rpms': kwargs.get('instRpms', unitPackages(target: distro) + ' daos-client-tests'), + 'image_version': kwargs.get('imageVersion', 'el9.7'), + 'prov_env_vars': kwargs.get('provEnvVars', '') + ] + Map unitTestPostArgs = kwargs.get('unitTestPostArgs', null) ?: [:] + Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:] + + return { + stage("${name}") { + println("[${name}] Starting stage: kwargs keys=${kwargs.keySet()}") + + if (!runStage) { + println("[${name}] Stage skipped by runStage=false") + Utils.markStageSkippedForConditional("${name}") + return + } + + node(label) { + // Ensure access to any branch provisioning scripts exist + if (testBranch) { + println("[${name}] Check out '${testBranch}' from version control") + checkoutScm( + url: 'https://github.com/daos-stack/daos.git', + branch: testBranch, + withSubmodules: false, + pruneStaleBranch: true) + } else { + println("[${name}] Check out branch from version control") + checkoutScm(pruneStaleBranch: true) + } + + try { + println("[${name}] Running unitTest() on ${label}") + jobStatusUpdate(jobStatus, name, unitTest(unitTestArgs)) + } catch (Exception e) { + println("[${name}] Caught exception: ${e}") + jobStatusUpdate(jobStatus, name, 'FAILURE') + throw e + } finally { + println("[${name}] Running unitTestPost()") + unitTestPost(unitTestPostArgs) + println("[${name}] Running archiveArtifacts()") + archiveArtifacts(archiveArtifactsArgs) + jobStatusUpdate(jobStatus, name) + } + } + println("[${name}] Finished with ${jobStatus}") + } + } +} From ffc879a86b3143e03e90fe73dda6303ab86a041f Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 11:55:46 -0400 Subject: [PATCH 07/36] Updates Signed-off-by: Phil Henderson --- vars/scriptedTestRpmStage.groovy | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index b67e2174e..7b67c6467 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -14,10 +14,11 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * testBranch if specified, checkout sources from this branch before running tests * jobStatus Map of status for each stage in the job/build * instRepos testRpm() inst_repos argument; defaults to daosRepos() - * daosPkgVersion testRpm() daos_pkg_version argument; defaults to daosPackagesVersion(next_version()) + * daosPkgVersion testRpm() daos_pkg_version argument; defaults to + * daosPackagesVersion(next_version()) * instRpms testRpm() inst_rpms argument; defaults to 'mercury-libfabric' * ignoreFailure testRpm() ignore_failure argument; defaults to false - * nodeList comma-delimited list of nodes; defaults to env.NODELIST + * alwaysScript script to run always after the test stage; defaults to ''. * @return a scripted stage to run in a pipeline */ Map call(Map kwargs = [:]) { @@ -34,7 +35,7 @@ Map call(Map kwargs = [:]) { inst_rpms: kwargs.get('instRpms', 'mercury-libfabric'), ignore_failure: kwargs.get('ignoreFailure', false) ] - String nodeList = kwargs.get('nodeList', env.NODELIST) + String alwaysScript = kwargs.get('alwaysScript', '') return { stage("${name}") { @@ -68,17 +69,11 @@ Map call(Map kwargs = [:]) { jobStatusUpdate(jobStatus, name, 'FAILURE') throw e } finally { + if (alwaysScript) { + sh(script: alwaysScript, label: "Running ${alwaysScript}") + } println("[${name}] Running archiveArtifacts()") - // Extract first node from comma-delimited list - String firstNode = nodeList.split(',')[0].trim() - sh label: 'Fetch and stage artifacts', - script: "hostname; ssh -i ci_key jenkins@${firstNode}" + - " ls -ltar /tmp; mkdir -p \"${name}\" && " + - "scp -i ci_key jenkins@${firstNode}:/tmp/" + - '{{suite_dmg,daos_{server_helper,{control,agent}}}.log,' + - 'daos_server.log.*}' + - " \"${name}/\"" - archiveArtifacts(artifacts: "${name}/**") + archiveArtifacts(archiveArtifactsArgs) jobStatusUpdate(jobStatus, name) } } From 5afdaeae75c38123803c4c9792685a96ce12959f Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 13:51:31 -0400 Subject: [PATCH 08/36] Updates Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 2 +- vars/scriptedDockerStage.groovy | 26 ++++++++++++++++---------- vars/scriptedTestRpmStage.groovy | 18 ++++++++++++------ vars/scriptedUnitTestStage.groovy | 18 ++++++++++++------ 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 406cc0bbe..d5a950248 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -135,7 +135,7 @@ Map call(Map kwargs = [:]) { jobStatusUpdate(job_status, name) } } + println("[${name}] Finished with ${job_status}") } - println("[${name}] Finished with ${job_status}") } } diff --git a/vars/scriptedDockerStage.groovy b/vars/scriptedDockerStage.groovy index 9921c8301..a8130a23e 100644 --- a/vars/scriptedDockerStage.groovy +++ b/vars/scriptedDockerStage.groovy @@ -103,7 +103,7 @@ Map call(Map kwargs = [:]) { } } } catch (Exception e) { - println("[${name}] Caught exception: ${e}") + println("[${name}] Caught exception in try: ${e}") if (sconsBuildArgs) { // Unsuccessful actions sh """if [ -f config.log ]; then @@ -115,16 +115,22 @@ Map call(Map kwargs = [:]) { throw e } finally { // Cleanup actions - if (buildRpms) { - uploadNewRPMs(uploadTarget, 'cleanup') - } - if (publishHtmlArgs) { - publishHTML(publishHtmlArgs) - } - if (archiveArtifactsArgs) { - archiveArtifacts(archiveArtifactsArgs) + try{ + if (buildRpms) { + uploadNewRPMs(uploadTarget, 'cleanup') + } + if (publishHtmlArgs) { + publishHTML(publishHtmlArgs) + } + if (archiveArtifactsArgs) { + archiveArtifacts(archiveArtifactsArgs) + } + jobStatusUpdate(jobStatus, name) + } catch (Exception e) { + println("[${name}] Caught exception in finally: ${e}") + jobStatusUpdate(jobStatus, name, 'FAILURE') + throw e } - jobStatusUpdate(jobStatus, name) } } println("[${name}] Finished with ${jobStatus}") diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index 7b67c6467..bed75277f 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -65,16 +65,22 @@ Map call(Map kwargs = [:]) { println("[${name}] Running testRpm() on ${label}") jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs)) } catch (Exception e) { - println("[${name}] Caught exception: ${e}") + println("[${name}] Caught exception in try: ${e}") jobStatusUpdate(jobStatus, name, 'FAILURE') throw e } finally { - if (alwaysScript) { - sh(script: alwaysScript, label: "Running ${alwaysScript}") + try { + if (alwaysScript) { + sh(script: alwaysScript, label: "Running ${alwaysScript}") + } + println("[${name}] Running archiveArtifacts()") + archiveArtifacts(archiveArtifactsArgs) + jobStatusUpdate(jobStatus, name) + } catch (Exception e) { + println("[${name}] Caught exception in finally: ${e}") + jobStatusUpdate(jobStatus, name, 'FAILURE') + throw e } - println("[${name}] Running archiveArtifacts()") - archiveArtifacts(archiveArtifactsArgs) - jobStatusUpdate(jobStatus, name) } } println("[${name}] Finished with ${jobStatus}") diff --git a/vars/scriptedUnitTestStage.groovy b/vars/scriptedUnitTestStage.groovy index edc2c2de1..afd13ab1f 100644 --- a/vars/scriptedUnitTestStage.groovy +++ b/vars/scriptedUnitTestStage.groovy @@ -83,15 +83,21 @@ Map call(Map kwargs = [:]) { println("[${name}] Running unitTest() on ${label}") jobStatusUpdate(jobStatus, name, unitTest(unitTestArgs)) } catch (Exception e) { - println("[${name}] Caught exception: ${e}") + println("[${name}] Caught exception in try: ${e}") jobStatusUpdate(jobStatus, name, 'FAILURE') throw e } finally { - println("[${name}] Running unitTestPost()") - unitTestPost(unitTestPostArgs) - println("[${name}] Running archiveArtifacts()") - archiveArtifacts(archiveArtifactsArgs) - jobStatusUpdate(jobStatus, name) + try { + println("[${name}] Running unitTestPost()") + unitTestPost(unitTestPostArgs) + println("[${name}] Running archiveArtifacts()") + archiveArtifacts(archiveArtifactsArgs) + jobStatusUpdate(jobStatus, name) + } catch (Exception e) { + println("[${name}] Caught exception in finally: ${e}") + jobStatusUpdate(jobStatus, name, 'FAILURE') + throw e + } } } println("[${name}] Finished with ${jobStatus}") From b22f7837b8f2c96ddd31fa2e66bf1ae514c17d54 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 14:47:05 -0400 Subject: [PATCH 09/36] Fixes. Signed-off-by: Phil Henderson --- vars/scriptedDockerStage.groovy | 3 +++ vars/scriptedTestRpmStage.groovy | 10 +++++++--- vars/scriptedUnitTestStage.groovy | 12 ++++++++---- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/vars/scriptedDockerStage.groovy b/vars/scriptedDockerStage.groovy index a8130a23e..b8630a876 100644 --- a/vars/scriptedDockerStage.groovy +++ b/vars/scriptedDockerStage.groovy @@ -117,12 +117,15 @@ Map call(Map kwargs = [:]) { // Cleanup actions try{ if (buildRpms) { + println("[${name}] Running uploadNewRPMs()") uploadNewRPMs(uploadTarget, 'cleanup') } if (publishHtmlArgs) { + println("[${name}] Running publishHTML()") publishHTML(publishHtmlArgs) } if (archiveArtifactsArgs) { + println("[${name}] Running archiveArtifacts()") archiveArtifacts(archiveArtifactsArgs) } jobStatusUpdate(jobStatus, name) diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index bed75277f..80dd82cb7 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -19,6 +19,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * instRpms testRpm() inst_rpms argument; defaults to 'mercury-libfabric' * ignoreFailure testRpm() ignore_failure argument; defaults to false * alwaysScript script to run always after the test stage; defaults to ''. + * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage * @return a scripted stage to run in a pipeline */ Map call(Map kwargs = [:]) { @@ -36,6 +37,7 @@ Map call(Map kwargs = [:]) { ignore_failure: kwargs.get('ignoreFailure', false) ] String alwaysScript = kwargs.get('alwaysScript', '') + Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:] return { stage("${name}") { @@ -71,10 +73,12 @@ Map call(Map kwargs = [:]) { } finally { try { if (alwaysScript) { - sh(script: alwaysScript, label: "Running ${alwaysScript}") + sh(script: alwaysScript, label: "Running alwaysScript: ${alwaysScript}") + } + if (archiveArtifactsArgs) { + println("[${name}] Running archiveArtifacts()") + archiveArtifacts(archiveArtifactsArgs) } - println("[${name}] Running archiveArtifacts()") - archiveArtifacts(archiveArtifactsArgs) jobStatusUpdate(jobStatus, name) } catch (Exception e) { println("[${name}] Caught exception in finally: ${e}") diff --git a/vars/scriptedUnitTestStage.groovy b/vars/scriptedUnitTestStage.groovy index afd13ab1f..af56cc36d 100644 --- a/vars/scriptedUnitTestStage.groovy +++ b/vars/scriptedUnitTestStage.groovy @@ -88,10 +88,14 @@ Map call(Map kwargs = [:]) { throw e } finally { try { - println("[${name}] Running unitTestPost()") - unitTestPost(unitTestPostArgs) - println("[${name}] Running archiveArtifacts()") - archiveArtifacts(archiveArtifactsArgs) + if (unitTestPostArgs) { + println("[${name}] Running unitTestPost()") + unitTestPost(unitTestPostArgs) + } + if (archiveArtifactsArgs) { + println("[${name}] Running archiveArtifacts()") + archiveArtifacts(archiveArtifactsArgs) + } jobStatusUpdate(jobStatus, name) } catch (Exception e) { println("[${name}] Caught exception in finally: ${e}") From 89f8bc17ab68524e3b20af832b388b5003bbf0d6 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 2 Jul 2026 18:41:21 -0400 Subject: [PATCH 10/36] Support passing in the distro arg to provisionNodes Signed-off-by: Phil Henderson --- vars/scriptedTestRpmStage.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index 80dd82cb7..c95338ed0 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -13,6 +13,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * label test stage default cluster label * testBranch if specified, checkout sources from this branch before running tests * jobStatus Map of status for each stage in the job/build + * imageVersion testRpm() target argument; defaults to '' * instRepos testRpm() inst_repos argument; defaults to daosRepos() * daosPkgVersion testRpm() daos_pkg_version argument; defaults to * daosPackagesVersion(next_version()) @@ -30,6 +31,7 @@ Map call(Map kwargs = [:]) { String testBranch = kwargs.get('testBranch', '') Map jobStatus = kwargs.get('jobStatus', null) ?: [:] Map testRpmArgs = [ + target: kwargs.get('imageVersion', ''), inst_repos: kwargs.get('instRepos', daosRepos()), daos_pkg_version: kwargs.get( 'daosPkgVersion', daosPackagesVersion(kwargs.get('next_version', null))), From efb52c0c2933c501380c7acb91f20a531bea2b87 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 7 Jul 2026 15:05:00 -0400 Subject: [PATCH 11/36] Remove unused code. Signed-off-by: Phil Henderson --- vars/scriptedDockerStage.groovy | 142 -------------------------------- 1 file changed, 142 deletions(-) delete mode 100644 vars/scriptedDockerStage.groovy diff --git a/vars/scriptedDockerStage.groovy b/vars/scriptedDockerStage.groovy deleted file mode 100644 index b8630a876..000000000 --- a/vars/scriptedDockerStage.groovy +++ /dev/null @@ -1,142 +0,0 @@ -// vars/scriptedDockerStage.groovy - -import org.jenkinsci.plugins.pipeline.modeldefinition.Utils - -/** - * scriptedDockerStage - * - * Get a build stage in scripted syntax. - * - * @param kwargs Map containing the following optional arguments (empty strings yield defaults): - * name the build stage name - * runStage optional additional condition to determine if the stage runs - * jobStatus Map of status for each stage in the job/build - * dockerTag the docker image tag to use for the build - * dockerBuildArgs optional docker build arguments - * buildRpms whether or not to build rpms; defaults to true - * distro the shorthand distro name; defaults to 'el8' - * rpmDistro the distro to use for rpm building; defaults to distro - * release the DAOS RPM release value to use; defaults to env.DAOS_RELVAL - * compiler the compiler to use; defaults to 'gcc' - * sconsBuildArgs optional scons build arguments - * valgrindSconsBuildArgs optional scons build arguments for valgrind build - * artifacts optional artifacts name to archive; defaults to - * "config.log-${distro}-${compiler}" - * uploadTarget the distro to use when uploading rpms; defaults to distro - * installScript optional script to run to install rpms; defaults to '' - * publishHtmlArgs optional arguments to pass to publishHTML() - * archiveArtifactsArgs optional arguments to pass to archiveArtifacts() - * @return a scripted stage to run in a pipeline - */ -Map call(Map kwargs = [:]) { - String name = kwargs.get('name', 'Unknown Docker Stage') - Boolean runStage = kwargs.get('runStage', true) - Map jobStatus = kwargs.get('jobStatus', [:]) - String dockerTag = jobStatusKey("build-${uploadTarget}-${compiler}").toLowerCase() - String dockerBuildArgs = kwargs.get('dockerBuildArgs', '') - - // Build stage parameters - Boolean buildRpms = kwargs.get('buildRpms', true) - String distro = kwargs.get('distro', 'el8') - String rpmDistro = kwargs.get('rpmDistro', distro) - String release = kwargs.get('release', env.DAOS_RELVAL) - String compiler = kwargs.get('compiler', 'gcc') - Map sconsBuildArgs = kwargs.get('sconsBuildArgs', [:]) - Map valgrindSconsBuildArgs = kwargs.get('valgrindSconsBuildArgs', [:]) - String artifacts = kwargs.get('artifacts', "config.log-${distro}-${compiler}") - String uploadTarget = kwargs.get('uploadTarget', distro) - String bullseye = 'false' - if (compiler == 'covc') { - bullseye = 'true' - } - - // Summary stage parameters - String installScript = kwargs.get('installScript', '') - Map publishHtmlArgs = kwargs.get('publishHtmlArgs', [:]) - Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', [:]) - - return { - stage("${name}") { - if (!runStage) { - println("[${name}] Marking docker stage as skipped") - Utils.markStageSkippedForConditional("${name}") - return - } - node('docker_runner') { - println("[${name}] Check out from version control") - checkoutScm(pruneStaleBranch: true) - - def dockerImage = docker.build(dockerTag, dockerBuildArgs) - try { - dockerImage.inside() { - if (buildRpms) { - sh label: 'Install RPMs', - script: "./ci/rpm/install_deps.sh ${rpmDistro} ${release} ${bullseye}" - // Avoid interpolation of sensitive environment variables - sh label: 'Build deps', - script: "./ci/rpm/build_deps.sh ${bullseye}" + ' ${BULLSEYE_KEY}' - } - if (sconsBuildArgs) { - job_step_update(sconsBuild(sconsBuildArgs)) - } - if (valgrindSconsBuildArgs) { - // For non-release builds, create a separate build with the valgrind - // tag for NLT memcheck testing. This is necessary to avoid problems - // caused by valgrind being confused by the Go runtime. We don't want - // to use the valgrind build for normal testing because it is much - // slower. BUILD_TYPE=dev is set for PR/dev builds in sconsArgs(), and - // TARGET_TYPE=release is used to select pre-built cached prerequisites. - job_step_update(sconsBuild(valgrindSconsBuildArgs)) - sh label: 'Stash valgrind install tree for NLT', - script: 'tar -C / -cf opt-daos-valgrind.tar opt/daos' - stash(name: 'opt-daos-valgrind', includes: 'opt-daos-valgrind.tar') - } - if (buildRpms) { - sh label: 'Generate RPMs', - script: "./ci/rpm/gen_rpms.sh ${rpmDistro} ${release} ${bullseye}" - // Success actions - uploadNewRPMs(uploadTarget, 'success') - } - if (installScript) { - sh label: 'Install RPMs', - script: "${installScript} ${distro}" - } - } - } catch (Exception e) { - println("[${name}] Caught exception in try: ${e}") - if (sconsBuildArgs) { - // Unsuccessful actions - sh """if [ -f config.log ]; then - mv config.log ${artifacts} - fi""" - archiveArtifacts artifacts: "${artifacts}", allowEmptyArchive: true - } - jobStatusUpdate(jobStatus, name, 'FAILURE') - throw e - } finally { - // Cleanup actions - try{ - if (buildRpms) { - println("[${name}] Running uploadNewRPMs()") - uploadNewRPMs(uploadTarget, 'cleanup') - } - if (publishHtmlArgs) { - println("[${name}] Running publishHTML()") - publishHTML(publishHtmlArgs) - } - if (archiveArtifactsArgs) { - println("[${name}] Running archiveArtifacts()") - archiveArtifacts(archiveArtifactsArgs) - } - jobStatusUpdate(jobStatus, name) - } catch (Exception e) { - println("[${name}] Caught exception in finally: ${e}") - jobStatusUpdate(jobStatus, name, 'FAILURE') - throw e - } - } - } - println("[${name}] Finished with ${jobStatus}") - } - } -} From 633079a9dadad2da0c7971081fb183460a441fae Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 8 Jul 2026 23:24:01 -0400 Subject: [PATCH 12/36] Cleanup defaults. Signed-off-by: Phil Henderson --- vars/scriptedTestRpmStage.groovy | 12 +++++------- vars/scriptedUnitTestStage.groovy | 21 ++++++++++----------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index c95338ed0..4c6eaf378 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -17,15 +17,16 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * instRepos testRpm() inst_repos argument; defaults to daosRepos() * daosPkgVersion testRpm() daos_pkg_version argument; defaults to * daosPackagesVersion(next_version()) - * instRpms testRpm() inst_rpms argument; defaults to 'mercury-libfabric' + * instRpms testRpm() inst_rpms argument; defaults to '' * ignoreFailure testRpm() ignore_failure argument; defaults to false - * alwaysScript script to run always after the test stage; defaults to ''. + * alwaysScript script to run always after the test stage, e.g. + * 'ci/rpm/test_daos_post.sh'; defaults to ''. * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage * @return a scripted stage to run in a pipeline */ Map call(Map kwargs = [:]) { // General parameters - String name = kwargs.get('name', 'Unknown Functional Test Stage') + String name = kwargs.get('name', 'Unknown Test RPM Stage') Boolean runStage = kwargs.get('runStage', true) as Boolean String label = kwargs.get('label') String testBranch = kwargs.get('testBranch', '') @@ -35,7 +36,7 @@ Map call(Map kwargs = [:]) { inst_repos: kwargs.get('instRepos', daosRepos()), daos_pkg_version: kwargs.get( 'daosPkgVersion', daosPackagesVersion(kwargs.get('next_version', null))), - inst_rpms: kwargs.get('instRpms', 'mercury-libfabric'), + inst_rpms: kwargs.get('instRpms', ''), ignore_failure: kwargs.get('ignoreFailure', false) ] String alwaysScript = kwargs.get('alwaysScript', '') @@ -43,14 +44,11 @@ Map call(Map kwargs = [:]) { return { stage("${name}") { - println("[${name}] Starting stage: kwargs keys=${kwargs.keySet()}") - if (!runStage) { println("[${name}] Stage skipped by runStage=false") Utils.markStageSkippedForConditional("${name}") return } - node(label) { // Ensure access to any branch provisioning scripts exist if (testBranch) { diff --git a/vars/scriptedUnitTestStage.groovy b/vars/scriptedUnitTestStage.groovy index af56cc36d..51dea5756 100644 --- a/vars/scriptedUnitTestStage.groovy +++ b/vars/scriptedUnitTestStage.groovy @@ -13,17 +13,19 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * label test stage default cluster label * testBranch if specified, checkout sources from this branch before running tests * jobStatus Map of status for each stage in the job/build - * target the target distro to use for the test stage; defaults to 'el9' + * distro the distro to use for the test stage, e.g. 'el9'; defaults to '' * timeoutTime unitTest() timeout time in minutes; defaults to 60 * instRepos unitTest() inst_repos argument; defaults to daosRepos() * testScript unitTest() test_script argument; defaults to '' * withValgrind unitTest() with_valgrind argument; defaults to '' - * alwaysScript unitTest() always_script argument; defaults to 'ci/unit/test_nlt_post.sh' + * alwaysScript unitTest() always_script argument, e.g. + * 'ci/unit/test_nlt_post.sh'; defaults to '' * testResults unitTest() test_results argument; defaults to 'nlt-junit.xml' * unstashOpt unitTest() unstash_opt argument; defaults to true * unstashTests unitTest() unstash_tests argument; defaults to false - * instRpms unitTest() inst_rpms argument; defaults to unitPackages(target: target) + ' daos-client-tests' - * imageVersion unitTest() image_version argument; defaults to 'el9.7' + * instRpms unitTest() inst_rpms argument; defaults to + * unitPackages(target: distro) + ' daos-client-tests' + * imageVersion unitTest() image_version argument, e.g 'el9.7'; defaults to '' * provEnvVars unitTest() prov_env_vars argument; defaults to '' * unitTestPostArgs Map of arguments to pass to unitTestPost() for the stage * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage @@ -31,12 +33,12 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils */ Map call(Map kwargs = [:]) { // General parameters - String name = kwargs.get('name', 'Unknown Functional Test Stage') + String name = kwargs.get('name', 'Unknown Unit Test Stage') Boolean runStage = kwargs.get('runStage', true) as Boolean String label = kwargs.get('label') String testBranch = kwargs.get('testBranch', '') Map jobStatus = kwargs.get('jobStatus', null) ?: [:] - String distro = kwargs.get('distro', 'el9') + String distro = kwargs.get('distro', '') // Unit Test stage parameters Map unitTestArgs = [ @@ -44,12 +46,12 @@ Map call(Map kwargs = [:]) { 'inst_repos': kwargs.get('instRepos', daosRepos(distro)), 'test_script': kwargs.get('testScript', ''), 'with_valgrind': kwargs.get('withValgrind', ''), - 'always_script': kwargs.get('alwaysScript', 'ci/unit/test_nlt_post.sh'), + 'always_script': kwargs.get('alwaysScript', ''), 'test_results': kwargs.get('testResults', 'nlt-junit.xml'), 'unstash_opt': kwargs.get('unstashOpt', true), 'unstash_tests': kwargs.get('unstashTests', false), 'inst_rpms': kwargs.get('instRpms', unitPackages(target: distro) + ' daos-client-tests'), - 'image_version': kwargs.get('imageVersion', 'el9.7'), + 'image_version': kwargs.get('imageVersion', ''), 'prov_env_vars': kwargs.get('provEnvVars', '') ] Map unitTestPostArgs = kwargs.get('unitTestPostArgs', null) ?: [:] @@ -57,14 +59,11 @@ Map call(Map kwargs = [:]) { return { stage("${name}") { - println("[${name}] Starting stage: kwargs keys=${kwargs.keySet()}") - if (!runStage) { println("[${name}] Stage skipped by runStage=false") Utils.markStageSkippedForConditional("${name}") return } - node(label) { // Ensure access to any branch provisioning scripts exist if (testBranch) { From 304ab81ea58ba25bf448e6f501e02a3aaa897d98 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 9 Jul 2026 03:00:37 -0400 Subject: [PATCH 13/36] Groovylint fixes Signed-off-by: Phil Henderson --- vars/scriptedTestRpmStage.groovy | 21 +++++++++++++++------ vars/scriptedUnitTestStage.groovy | 21 +++++++++++++++------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index 4c6eaf378..860469dc5 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable NestedBlockDepth */ // vars/scriptedTestRpmStage.groovy import org.jenkinsci.plugins.pipeline.modeldefinition.Utils @@ -23,7 +24,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * 'ci/rpm/test_daos_post.sh'; defaults to ''. * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage * @return a scripted stage to run in a pipeline - */ + */ Map call(Map kwargs = [:]) { // General parameters String name = kwargs.get('name', 'Unknown Test RPM Stage') @@ -63,13 +64,16 @@ Map call(Map kwargs = [:]) { checkoutScm(pruneStaleBranch: true) } + Throwable tryError = null try { println("[${name}] Running testRpm() on ${label}") jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs)) + /* groovylint-disable-next-line CatchException */ } catch (Exception e) { - println("[${name}] Caught exception in try: ${e}") + tryError = e + println("[${name}] Caught exception in try: ${tryError}") jobStatusUpdate(jobStatus, name, 'FAILURE') - throw e + throw tryError } finally { try { if (alwaysScript) { @@ -80,10 +84,15 @@ Map call(Map kwargs = [:]) { archiveArtifacts(archiveArtifactsArgs) } jobStatusUpdate(jobStatus, name) - } catch (Exception e) { - println("[${name}] Caught exception in finally: ${e}") + /* groovylint-disable-next-line CatchException */ + } catch (Exception finallyError) { + println("[${name}] Caught exception in finally: ${finallyError}") + /* groovylint-disable-next-line DuplicateStringLiteral */ jobStatusUpdate(jobStatus, name, 'FAILURE') - throw e + if (tryError == null) { + /* groovylint-disable-next-line ThrowExceptionFromFinallyBlock */ + throw finallyError + } } } } diff --git a/vars/scriptedUnitTestStage.groovy b/vars/scriptedUnitTestStage.groovy index 51dea5756..07ae6e166 100644 --- a/vars/scriptedUnitTestStage.groovy +++ b/vars/scriptedUnitTestStage.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable NestedBlockDepth */ // vars/scriptedUnitTestStage.groovy import org.jenkinsci.plugins.pipeline.modeldefinition.Utils @@ -30,7 +31,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * unitTestPostArgs Map of arguments to pass to unitTestPost() for the stage * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage * @return a scripted stage to run in a pipeline - */ + */ Map call(Map kwargs = [:]) { // General parameters String name = kwargs.get('name', 'Unknown Unit Test Stage') @@ -78,13 +79,16 @@ Map call(Map kwargs = [:]) { checkoutScm(pruneStaleBranch: true) } + Throwable tryError = null try { println("[${name}] Running unitTest() on ${label}") jobStatusUpdate(jobStatus, name, unitTest(unitTestArgs)) + /* groovylint-disable-next-line CatchException */ } catch (Exception e) { - println("[${name}] Caught exception in try: ${e}") + tryError = e + println("[${name}] Caught exception in try: ${tryError}") jobStatusUpdate(jobStatus, name, 'FAILURE') - throw e + throw tryError } finally { try { if (unitTestPostArgs) { @@ -96,10 +100,15 @@ Map call(Map kwargs = [:]) { archiveArtifacts(archiveArtifactsArgs) } jobStatusUpdate(jobStatus, name) - } catch (Exception e) { - println("[${name}] Caught exception in finally: ${e}") + /* groovylint-disable-next-line CatchException */ + } catch (Exception finallyError) { + println("[${name}] Caught exception in finally: ${finallyError}") + /* groovylint-disable-next-line DuplicateStringLiteral */ jobStatusUpdate(jobStatus, name, 'FAILURE') - throw e + if (tryError == null) { + /* groovylint-disable-next-line ThrowExceptionFromFinallyBlock */ + throw finallyError + } } } } From 8cf3e29e93628eac27406dde2a8b17fa034a454a Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 9 Jul 2026 16:43:45 -0400 Subject: [PATCH 14/36] Applying feedback. Signed-off-by: Phil Henderson --- vars/scriptedTestRpmStage.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index 860469dc5..a12edc540 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -10,9 +10,10 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * * @param kwargs Map containing the following optional arguments (empty strings yield defaults): * name test stage name - * runStage whether or not to run the test stage + * runStage whether or not to run the test stage; defaults to true * label test stage default cluster label - * testBranch if specified, checkout sources from this branch before running tests + * testBranch if specified, checkout sources from this branch before running tests; + * defaults to '' * jobStatus Map of status for each stage in the job/build * imageVersion testRpm() target argument; defaults to '' * instRepos testRpm() inst_repos argument; defaults to daosRepos() From 61a43af71b2f19b064e40df61a551e62d0799ef0 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 9 Jul 2026 17:56:09 -0400 Subject: [PATCH 15/36] Fix skipping functional test stage due to no tag match Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index d5a950248..d23901701 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -33,6 +33,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * job_status Map of status for each stage in the job/build * @return a scripted stage to run in a pipeline */ +/* groovylint-disable-next-line MethodSize */ Map call(Map kwargs = [:]) { String name = kwargs.get('name', 'Unknown Functional Test Stage') String pragma_suffix = kwargs.get('pragma_suffix') @@ -57,7 +58,7 @@ Map call(Map kwargs = [:]) { // to be skipped by the existing skipFunctionalTestStage logic, while new stages can use // runStage directly. Once all stages have been converted to use runStage, this should be // changed to a Boolean. - String runStage = kwargs.get('runStage', 'undefined').toString() + String runStage = kwargs.get('runStage', 'undefined') Map job_status = kwargs.get('job_status', [:]) @@ -69,12 +70,22 @@ Map call(Map kwargs = [:]) { String tags = getFunctionalTags( pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags) + /* groovylint-disable-next-line DuplicateStringLiteral */ if (runStage == 'false') { println("[${name}] Stage skipped by runStage=false") Utils.markStageSkippedForConditional("${name}") return - } else if (runStage == 'undefined') { - // To be removed once all stages have been converted to use runStage. + } + + if (runStage == 'true' && !testsInStage(tags)) { + println("[${name}] Stage skipped by no tests matching the '${tags}' tags") + Utils.markStageSkippedForConditional("${name}") + return + } + + // To be removed once all stages have been converted to use runStage. + /* groovylint-disable-next-line DuplicateStringLiteral */ + if (runStage == 'undefined') { Map skip_kwargs = [ 'tags': tags, 'pragma_suffix': pragma_suffix, @@ -86,10 +97,6 @@ Map call(Map kwargs = [:]) { Utils.markStageSkippedForConditional("${name}") return } - } else if (!testsInStage(tags)) { - println("[${name}] Stage skipped by no tests matching the '${tags}' tags") - Utils.markStageSkippedForConditional("${name}") - return } node(cachedCommitPragma("Test-label${pragma_suffix}", label)) { @@ -123,6 +130,7 @@ Map call(Map kwargs = [:]) { provider: provider)['ftest_arg'], test_function: 'runTestFunctionalV2'] if (node_count != null) { + /* groovylint-disable-next-line DuplicateStringLiteral */ ftestConfig['node_count'] = node_count } jobStatusUpdate( From 1a3cdefa2d916631bf534a5dc4fc32f127d663e7 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 9 Jul 2026 19:39:11 -0400 Subject: [PATCH 16/36] Update tag fit check for functional test stages Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 35 ++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index d23901701..9a0727a92 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -64,12 +64,6 @@ Map call(Map kwargs = [:]) { return { stage("${name}") { - // Get the tags for the stage. Use either the build parameter, commit pragma, or - // default tags. All tags are combined with the stage tags to ensure only tests that - // 'fit' the cluster will be run. - String tags = getFunctionalTags( - pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags) - /* groovylint-disable-next-line DuplicateStringLiteral */ if (runStage == 'false') { println("[${name}] Stage skipped by runStage=false") @@ -77,6 +71,12 @@ Map call(Map kwargs = [:]) { return } + // Get the tags for the stage. Use either the build parameter, commit pragma, or + // default tags. All tags are combined with the stage tags to ensure only tests that + // 'fit' the cluster will be run. + String tags = getFunctionalTags( + pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags) + println("[${name}] Functional test tags for stage: ${tags}") if (runStage == 'true' && !testsInStage(tags)) { println("[${name}] Stage skipped by no tests matching the '${tags}' tags") Utils.markStageSkippedForConditional("${name}") @@ -112,6 +112,7 @@ Map call(Map kwargs = [:]) { checkoutScm(pruneStaleBranch: true) } + Throwable tryError = null try { println("[${name}] Running functionalTest() on ${label} with tags=${tags}") Map ftestConfig = [ @@ -137,10 +138,26 @@ Map call(Map kwargs = [:]) { job_status, name, functionalTest(ftestConfig)) + /* groovylint-disable-next-line CatchException */ + } catch (Exception e) { + tryError = e + println("[${name}] Caught exception in try: ${tryError}") + jobStatusUpdate(job_status, name, 'FAILURE') + throw tryError } finally { - println("[${name}] Running functionalTestPostV2()") - functionalTestPostV2() - jobStatusUpdate(job_status, name) + try{ + println("[${name}] Running functionalTestPostV2()") + functionalTestPostV2() + jobStatusUpdate(job_status, name) + } catch (Exception finallyError) { + println("[${name}] Caught exception in finally: ${finallyError}") + /* groovylint-disable-next-line DuplicateStringLiteral */ + jobStatusUpdate(job_status, name, 'FAILURE') + if (tryError == null) { + /* groovylint-disable-next-line ThrowExceptionFromFinallyBlock */ + throw finallyError + } + } } } println("[${name}] Finished with ${job_status}") From 1950b3aaca47f4fd4bca2a9dabb74a312ff29238 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 9 Jul 2026 20:22:07 -0400 Subject: [PATCH 17/36] Debug Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 9a0727a92..8dce7a70c 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -64,6 +64,8 @@ Map call(Map kwargs = [:]) { return { stage("${name}") { + println("[${name}] runStage: ${runStage}") + /* groovylint-disable-next-line DuplicateStringLiteral */ if (runStage == 'false') { println("[${name}] Stage skipped by runStage=false") @@ -77,7 +79,9 @@ Map call(Map kwargs = [:]) { String tags = getFunctionalTags( pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags) println("[${name}] Functional test tags for stage: ${tags}") - if (runStage == 'true' && !testsInStage(tags)) { + Boolean tagMatch = testsInStage(tags) + println("[${name}] Functional test tags match the stage: ${tagMatch}") + if (runStage == 'true' && !tagMatch) { println("[${name}] Stage skipped by no tests matching the '${tags}' tags") Utils.markStageSkippedForConditional("${name}") return From 3e2f4fcec659cd3da898d8815901dc998d0843c2 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Mon, 13 Jul 2026 10:21:21 -0400 Subject: [PATCH 18/36] Debug to figure out why testsInStage isn't working. Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/testsInStage.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vars/testsInStage.groovy b/vars/testsInStage.groovy index 96eca3951..6701cc91a 100644 --- a/vars/testsInStage.groovy +++ b/vars/testsInStage.groovy @@ -10,11 +10,13 @@ * @return boolean true if there are tests that match the tags */ boolean call(String tags) { + println("[${name}] Calling testsInStage(${tags})") if (env.UNIT_TEST && env.UNIT_TEST == 'true') { return true } /* groovylint-disable-next-line UnnecessaryGetter */ String verbose = isPr() ? '--verbose ' : '' + println("[${name}] Running script; verbose=${verbose}") return sh(label: 'Get test list', /* groovylint-disable-next-line GStringExpressionWithinString */ script: '''trap 'echo "Got an unhandled error, exiting as if a match was found"; exit 0' ERR From 037d0ddc14145bc96d08751797e10cc4b86a0d2a Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Mon, 13 Jul 2026 23:30:55 -0400 Subject: [PATCH 19/36] Fix debug Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/testsInStage.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/testsInStage.groovy b/vars/testsInStage.groovy index 6701cc91a..eaacd9bbc 100644 --- a/vars/testsInStage.groovy +++ b/vars/testsInStage.groovy @@ -10,13 +10,13 @@ * @return boolean true if there are tests that match the tags */ boolean call(String tags) { - println("[${name}] Calling testsInStage(${tags})") + println("[testsInStage] Calling testsInStage(${tags})") if (env.UNIT_TEST && env.UNIT_TEST == 'true') { return true } /* groovylint-disable-next-line UnnecessaryGetter */ String verbose = isPr() ? '--verbose ' : '' - println("[${name}] Running script; verbose=${verbose}") + println("[testsInStage] Running script; verbose=${verbose}") return sh(label: 'Get test list', /* groovylint-disable-next-line GStringExpressionWithinString */ script: '''trap 'echo "Got an unhandled error, exiting as if a match was found"; exit 0' ERR From 8d53241fcfe9f3a14f85f0850332ed29035e7f49 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 14 Jul 2026 09:18:35 -0400 Subject: [PATCH 20/36] Cleanup debug and apply feedback Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 25 +++++++++++++---------- vars/scriptedTestRpmStage.groovy | 27 ++++++++++++++++--------- vars/scriptedUnitTestStage.groovy | 32 +++++++++++++++++------------- vars/testsInStage.groovy | 2 -- 4 files changed, 50 insertions(+), 36 deletions(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 8dce7a70c..c68d1a160 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -1,4 +1,4 @@ -/* groovylint-disable VariableName */ +/* groovylint-disable NestedBlockDepth, VariableName */ // vars/getFunctionalTestStage.groovy import org.jenkinsci.plugins.pipeline.modeldefinition.Utils @@ -35,7 +35,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils */ /* groovylint-disable-next-line MethodSize */ Map call(Map kwargs = [:]) { - String name = kwargs.get('name', 'Unknown Functional Test Stage') + String name = kwargs.get('name', '') String pragma_suffix = kwargs.get('pragma_suffix') String label = kwargs.get('label') String next_version = kwargs.get('next_version', null) @@ -62,6 +62,10 @@ Map call(Map kwargs = [:]) { Map job_status = kwargs.get('job_status', [:]) + if (!name) { + error("getFunctionalTestStage() requires a stage 'name' argument") + } + return { stage("${name}") { println("[${name}] runStage: ${runStage}") @@ -78,14 +82,6 @@ Map call(Map kwargs = [:]) { // 'fit' the cluster will be run. String tags = getFunctionalTags( pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags) - println("[${name}] Functional test tags for stage: ${tags}") - Boolean tagMatch = testsInStage(tags) - println("[${name}] Functional test tags match the stage: ${tagMatch}") - if (runStage == 'true' && !tagMatch) { - println("[${name}] Stage skipped by no tests matching the '${tags}' tags") - Utils.markStageSkippedForConditional("${name}") - return - } // To be removed once all stages have been converted to use runStage. /* groovylint-disable-next-line DuplicateStringLiteral */ @@ -103,6 +99,12 @@ Map call(Map kwargs = [:]) { } } + if (runStage == 'true' && !testsInStage(tags)) { + println("[${name}] Stage skipped by no tests matching the '${tags}' tags") + Utils.markStageSkippedForConditional("${name}") + return + } + node(cachedCommitPragma("Test-label${pragma_suffix}", label)) { // Ensure access to any branch provisioning scripts exist println("[${name}] Check out '${base_branch}' from version control") @@ -149,10 +151,11 @@ Map call(Map kwargs = [:]) { jobStatusUpdate(job_status, name, 'FAILURE') throw tryError } finally { - try{ + try { println("[${name}] Running functionalTestPostV2()") functionalTestPostV2() jobStatusUpdate(job_status, name) + /* groovylint-disable-next-line CatchException */ } catch (Exception finallyError) { println("[${name}] Caught exception in finally: ${finallyError}") /* groovylint-disable-next-line DuplicateStringLiteral */ diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index a12edc540..2b8139df3 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -28,22 +28,31 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils */ Map call(Map kwargs = [:]) { // General parameters - String name = kwargs.get('name', 'Unknown Test RPM Stage') + String name = kwargs.get('name', '') Boolean runStage = kwargs.get('runStage', true) as Boolean String label = kwargs.get('label') String testBranch = kwargs.get('testBranch', '') Map jobStatus = kwargs.get('jobStatus', null) ?: [:] - Map testRpmArgs = [ - target: kwargs.get('imageVersion', ''), - inst_repos: kwargs.get('instRepos', daosRepos()), - daos_pkg_version: kwargs.get( - 'daosPkgVersion', daosPackagesVersion(kwargs.get('next_version', null))), - inst_rpms: kwargs.get('instRpms', ''), - ignore_failure: kwargs.get('ignoreFailure', false) - ] + Map testRpmArgs = kwargs.get('testRpmArgs', null) ?: [:] String alwaysScript = kwargs.get('alwaysScript', '') Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:] + if (!name) { + error("scriptedTestRpmStage() requires a stage 'name' argument") + } + + // Add defaults for any missing testRpm() arguments + if (!testRpmArgs.containsKey('inst_repos')) { + // If inst_repos is not specified, use the default daosRepos() for the specified distro + /* groovylint-disable-next-line DuplicateStringLiteral */ + testRpmArgs['inst_repos'] = daosRepos() + } + if (!testRpmArgs.containsKey('daos_pkg_version')) { + // If daos_pkg_version is not specified, use the default daosPackagesVersion() + /* groovylint-disable-next-line DuplicateStringLiteral */ + testRpmArgs['daos_pkg_version'] = daosPackagesVersion(kwargs.get('next_version', null)) + } + return { stage("${name}") { if (!runStage) { diff --git a/vars/scriptedUnitTestStage.groovy b/vars/scriptedUnitTestStage.groovy index 07ae6e166..f790bc456 100644 --- a/vars/scriptedUnitTestStage.groovy +++ b/vars/scriptedUnitTestStage.groovy @@ -34,7 +34,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils */ Map call(Map kwargs = [:]) { // General parameters - String name = kwargs.get('name', 'Unknown Unit Test Stage') + String name = kwargs.get('name', '') Boolean runStage = kwargs.get('runStage', true) as Boolean String label = kwargs.get('label') String testBranch = kwargs.get('testBranch', '') @@ -42,22 +42,26 @@ Map call(Map kwargs = [:]) { String distro = kwargs.get('distro', '') // Unit Test stage parameters - Map unitTestArgs = [ - 'timeout_time': kwargs.get('timeoutTime', 60), - 'inst_repos': kwargs.get('instRepos', daosRepos(distro)), - 'test_script': kwargs.get('testScript', ''), - 'with_valgrind': kwargs.get('withValgrind', ''), - 'always_script': kwargs.get('alwaysScript', ''), - 'test_results': kwargs.get('testResults', 'nlt-junit.xml'), - 'unstash_opt': kwargs.get('unstashOpt', true), - 'unstash_tests': kwargs.get('unstashTests', false), - 'inst_rpms': kwargs.get('instRpms', unitPackages(target: distro) + ' daos-client-tests'), - 'image_version': kwargs.get('imageVersion', ''), - 'prov_env_vars': kwargs.get('provEnvVars', '') - ] + Map unitTestArgs = kwargs.get('unitTestArgs', null) ?: [:] Map unitTestPostArgs = kwargs.get('unitTestPostArgs', null) ?: [:] Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:] + if (!name) { + error("scriptedUnitTestStage() requires a stage 'name' argument") + } + + // Add defaults for any missing unitTest() arguments + if (!unitTestArgs.containsKey('inst_repos')) { + // If inst_repos is not specified, use the default daosRepos() for the specified distro + /* groovylint-disable-next-line DuplicateStringLiteral */ + unitTestArgs['inst_repos'] = daosRepos(distro) + } + if (!unitTestArgs.containsKey('inst_rpms')) { + // If inst_rpms is not specified, use the default unitPackages() for the specified distro + /* groovylint-disable-next-line DuplicateStringLiteral */ + unitTestArgs['inst_rpms'] = unitPackages(target: distro) + ' daos-client-tests' + } + return { stage("${name}") { if (!runStage) { diff --git a/vars/testsInStage.groovy b/vars/testsInStage.groovy index eaacd9bbc..96eca3951 100644 --- a/vars/testsInStage.groovy +++ b/vars/testsInStage.groovy @@ -10,13 +10,11 @@ * @return boolean true if there are tests that match the tags */ boolean call(String tags) { - println("[testsInStage] Calling testsInStage(${tags})") if (env.UNIT_TEST && env.UNIT_TEST == 'true') { return true } /* groovylint-disable-next-line UnnecessaryGetter */ String verbose = isPr() ? '--verbose ' : '' - println("[testsInStage] Running script; verbose=${verbose}") return sh(label: 'Get test list', /* groovylint-disable-next-line GStringExpressionWithinString */ script: '''trap 'echo "Got an unhandled error, exiting as if a match was found"; exit 0' ERR From 9deead5c34ce915c1fdb78eb51bf37cb5174e449 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 14 Jul 2026 09:30:52 -0400 Subject: [PATCH 21/36] Cleanup docstrings Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 3 +-- vars/scriptedTestRpmStage.groovy | 9 +++------ vars/scriptedUnitTestStage.groovy | 14 +------------- 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index c68d1a160..452bf952e 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -68,8 +68,6 @@ Map call(Map kwargs = [:]) { return { stage("${name}") { - println("[${name}] runStage: ${runStage}") - /* groovylint-disable-next-line DuplicateStringLiteral */ if (runStage == 'false') { println("[${name}] Stage skipped by runStage=false") @@ -99,6 +97,7 @@ Map call(Map kwargs = [:]) { } } + // Verify tags are valid for the stage and that there are tests to run. if (runStage == 'true' && !testsInStage(tags)) { println("[${name}] Stage skipped by no tests matching the '${tags}' tags") Utils.markStageSkippedForConditional("${name}") diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index 2b8139df3..c84f9cf11 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -15,15 +15,12 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * testBranch if specified, checkout sources from this branch before running tests; * defaults to '' * jobStatus Map of status for each stage in the job/build - * imageVersion testRpm() target argument; defaults to '' - * instRepos testRpm() inst_repos argument; defaults to daosRepos() - * daosPkgVersion testRpm() daos_pkg_version argument; defaults to - * daosPackagesVersion(next_version()) - * instRpms testRpm() inst_rpms argument; defaults to '' - * ignoreFailure testRpm() ignore_failure argument; defaults to false + * testRpmArgs Map of arguments to pass to testRpm() for the stage * alwaysScript script to run always after the test stage, e.g. * 'ci/rpm/test_daos_post.sh'; defaults to ''. * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage + * next_version next daos package version to pass to daosPackagesVersion() if + * testRpmArgs does not specify a daos_pkg_version; defaults to null * @return a scripted stage to run in a pipeline */ Map call(Map kwargs = [:]) { diff --git a/vars/scriptedUnitTestStage.groovy b/vars/scriptedUnitTestStage.groovy index f790bc456..704c411da 100644 --- a/vars/scriptedUnitTestStage.groovy +++ b/vars/scriptedUnitTestStage.groovy @@ -15,19 +15,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * testBranch if specified, checkout sources from this branch before running tests * jobStatus Map of status for each stage in the job/build * distro the distro to use for the test stage, e.g. 'el9'; defaults to '' - * timeoutTime unitTest() timeout time in minutes; defaults to 60 - * instRepos unitTest() inst_repos argument; defaults to daosRepos() - * testScript unitTest() test_script argument; defaults to '' - * withValgrind unitTest() with_valgrind argument; defaults to '' - * alwaysScript unitTest() always_script argument, e.g. - * 'ci/unit/test_nlt_post.sh'; defaults to '' - * testResults unitTest() test_results argument; defaults to 'nlt-junit.xml' - * unstashOpt unitTest() unstash_opt argument; defaults to true - * unstashTests unitTest() unstash_tests argument; defaults to false - * instRpms unitTest() inst_rpms argument; defaults to - * unitPackages(target: distro) + ' daos-client-tests' - * imageVersion unitTest() image_version argument, e.g 'el9.7'; defaults to '' - * provEnvVars unitTest() prov_env_vars argument; defaults to '' + * unitTestArgs Map of arguments to pass to unitTest; defaults to an empty Map. * unitTestPostArgs Map of arguments to pass to unitTestPost() for the stage * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage * @return a scripted stage to run in a pipeline From 0162051eb3f92cb52c3abeabeb3404c1c389b1ed Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 14 Jul 2026 10:48:03 -0400 Subject: [PATCH 22/36] Applying feedback Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/scriptedTestRpmStage.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index c84f9cf11..9bb6570a1 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -16,7 +16,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * defaults to '' * jobStatus Map of status for each stage in the job/build * testRpmArgs Map of arguments to pass to testRpm() for the stage - * alwaysScript script to run always after the test stage, e.g. + * postScript script to run after the test stage, e.g. * 'ci/rpm/test_daos_post.sh'; defaults to ''. * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage * next_version next daos package version to pass to daosPackagesVersion() if @@ -31,7 +31,7 @@ Map call(Map kwargs = [:]) { String testBranch = kwargs.get('testBranch', '') Map jobStatus = kwargs.get('jobStatus', null) ?: [:] Map testRpmArgs = kwargs.get('testRpmArgs', null) ?: [:] - String alwaysScript = kwargs.get('alwaysScript', '') + String postScript = kwargs.get('postScript', '') Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:] if (!name) { @@ -83,8 +83,8 @@ Map call(Map kwargs = [:]) { throw tryError } finally { try { - if (alwaysScript) { - sh(script: alwaysScript, label: "Running alwaysScript: ${alwaysScript}") + if (postScript) { + sh(script: postScript, label: "Running postScript: ${postScript}") } if (archiveArtifactsArgs) { println("[${name}] Running archiveArtifacts()") From 6b92bcf4b956b3c52d76b2d9137a70197d327059 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 14 Jul 2026 11:15:16 -0400 Subject: [PATCH 23/36] Ensure methods relying on the stage name run in the stage Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/scriptedTestRpmStage.groovy | 24 ++++++++++++------------ vars/scriptedUnitTestStage.groovy | 23 +++++++++++------------ 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index 9bb6570a1..35796c6da 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -38,18 +38,6 @@ Map call(Map kwargs = [:]) { error("scriptedTestRpmStage() requires a stage 'name' argument") } - // Add defaults for any missing testRpm() arguments - if (!testRpmArgs.containsKey('inst_repos')) { - // If inst_repos is not specified, use the default daosRepos() for the specified distro - /* groovylint-disable-next-line DuplicateStringLiteral */ - testRpmArgs['inst_repos'] = daosRepos() - } - if (!testRpmArgs.containsKey('daos_pkg_version')) { - // If daos_pkg_version is not specified, use the default daosPackagesVersion() - /* groovylint-disable-next-line DuplicateStringLiteral */ - testRpmArgs['daos_pkg_version'] = daosPackagesVersion(kwargs.get('next_version', null)) - } - return { stage("${name}") { if (!runStage) { @@ -57,6 +45,18 @@ Map call(Map kwargs = [:]) { Utils.markStageSkippedForConditional("${name}") return } + + // Add defaults for any missing testRpm() arguments + if (!testRpmArgs.containsKey('inst_repos')) { + /* groovylint-disable-next-line DuplicateStringLiteral */ + testRpmArgs['inst_repos'] = daosRepos() + } + if (!testRpmArgs.containsKey('daos_pkg_version')) { + /* groovylint-disable-next-line DuplicateStringLiteral */ + testRpmArgs['daos_pkg_version'] = daosPackagesVersion( + kwargs.get('next_version', null)) + } + node(label) { // Ensure access to any branch provisioning scripts exist if (testBranch) { diff --git a/vars/scriptedUnitTestStage.groovy b/vars/scriptedUnitTestStage.groovy index 704c411da..9f8ec0769 100644 --- a/vars/scriptedUnitTestStage.groovy +++ b/vars/scriptedUnitTestStage.groovy @@ -38,18 +38,6 @@ Map call(Map kwargs = [:]) { error("scriptedUnitTestStage() requires a stage 'name' argument") } - // Add defaults for any missing unitTest() arguments - if (!unitTestArgs.containsKey('inst_repos')) { - // If inst_repos is not specified, use the default daosRepos() for the specified distro - /* groovylint-disable-next-line DuplicateStringLiteral */ - unitTestArgs['inst_repos'] = daosRepos(distro) - } - if (!unitTestArgs.containsKey('inst_rpms')) { - // If inst_rpms is not specified, use the default unitPackages() for the specified distro - /* groovylint-disable-next-line DuplicateStringLiteral */ - unitTestArgs['inst_rpms'] = unitPackages(target: distro) + ' daos-client-tests' - } - return { stage("${name}") { if (!runStage) { @@ -57,6 +45,17 @@ Map call(Map kwargs = [:]) { Utils.markStageSkippedForConditional("${name}") return } + + // Add defaults for any missing unitTest() arguments + if (!unitTestArgs.containsKey('inst_repos')) { + /* groovylint-disable-next-line DuplicateStringLiteral */ + unitTestArgs['inst_repos'] = daosRepos(distro) + } + if (!unitTestArgs.containsKey('inst_rpms')) { + /* groovylint-disable-next-line DuplicateStringLiteral */ + unitTestArgs['inst_rpms'] = unitPackages(target: distro) + ' daos-client-tests' + } + node(label) { // Ensure access to any branch provisioning scripts exist if (testBranch) { From 3c8c2dca512a280c4049395c9ee1bbb97898fc62 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 14 Jul 2026 13:29:31 -0400 Subject: [PATCH 24/36] Adding info message to testsInStage(). Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/testsInStage.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/vars/testsInStage.groovy b/vars/testsInStage.groovy index 96eca3951..72305bae0 100644 --- a/vars/testsInStage.groovy +++ b/vars/testsInStage.groovy @@ -10,6 +10,7 @@ * @return boolean true if there are tests that match the tags */ boolean call(String tags) { + println("[${env.STAGE_NAME}] Determining if tests w/ '${tags}' tags exist for this stage") if (env.UNIT_TEST && env.UNIT_TEST == 'true') { return true } From 8130ede3c1904afd9982beb4a56f2ec33b096f59 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 15 Jul 2026 09:48:02 -0400 Subject: [PATCH 25/36] Adding clarifications to docstrings. Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/scriptedUnitTestStage.groovy | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vars/scriptedUnitTestStage.groovy b/vars/scriptedUnitTestStage.groovy index 9f8ec0769..a332cf457 100644 --- a/vars/scriptedUnitTestStage.groovy +++ b/vars/scriptedUnitTestStage.groovy @@ -14,10 +14,13 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * label test stage default cluster label * testBranch if specified, checkout sources from this branch before running tests * jobStatus Map of status for each stage in the job/build - * distro the distro to use for the test stage, e.g. 'el9'; defaults to '' + * distro the distro to use for daosRepos() and unitPackages() when providing + * default arguments in unitTestArgs, e.g. 'el9'; defaults to '' * unitTestArgs Map of arguments to pass to unitTest; defaults to an empty Map. - * unitTestPostArgs Map of arguments to pass to unitTestPost() for the stage - * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage + * unitTestPostArgs Map of arguments to pass to unitTestPost() for the stage; defaults to + * an empty Map. + * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage; defaults + * to an empty Map. * @return a scripted stage to run in a pipeline */ Map call(Map kwargs = [:]) { From 495d93a503e4c1564674fcdf7d5537359968dd17 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 15 Jul 2026 10:36:53 -0400 Subject: [PATCH 26/36] Collect return status from alwaysScript. Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/scriptedTestRpmStage.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index 35796c6da..2965185e3 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -16,7 +16,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * defaults to '' * jobStatus Map of status for each stage in the job/build * testRpmArgs Map of arguments to pass to testRpm() for the stage - * postScript script to run after the test stage, e.g. + * alwaysScript script to always run after the test stage, e.g. * 'ci/rpm/test_daos_post.sh'; defaults to ''. * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage * next_version next daos package version to pass to daosPackagesVersion() if @@ -31,7 +31,7 @@ Map call(Map kwargs = [:]) { String testBranch = kwargs.get('testBranch', '') Map jobStatus = kwargs.get('jobStatus', null) ?: [:] Map testRpmArgs = kwargs.get('testRpmArgs', null) ?: [:] - String postScript = kwargs.get('postScript', '') + String alwaysScript = kwargs.get('alwaysScript', '') Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:] if (!name) { @@ -83,8 +83,10 @@ Map call(Map kwargs = [:]) { throw tryError } finally { try { - if (postScript) { - sh(script: postScript, label: "Running postScript: ${postScript}") + if (alwaysScript) { + sh(script: alwaysScript, + label: "Running alwaysScript: ${alwaysScript}", + returnStatus: true) } if (archiveArtifactsArgs) { println("[${name}] Running archiveArtifacts()") From 8e3ed9dbbe79056a075a5690f89450b562104cbe Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 15 Jul 2026 17:23:25 -0400 Subject: [PATCH 27/36] Allow distro to be passed to daosRepos() Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/scriptedTestRpmStage.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vars/scriptedTestRpmStage.groovy b/vars/scriptedTestRpmStage.groovy index 2965185e3..35ad55bfe 100644 --- a/vars/scriptedTestRpmStage.groovy +++ b/vars/scriptedTestRpmStage.groovy @@ -17,8 +17,10 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils * jobStatus Map of status for each stage in the job/build * testRpmArgs Map of arguments to pass to testRpm() for the stage * alwaysScript script to always run after the test stage, e.g. - * 'ci/rpm/test_daos_post.sh'; defaults to ''. + * 'ci/rpm/test_daos_post.sh'; defaults to '' * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() for the stage + * distro the distro to pass to daosRepos() if testRpmArgs does not specify a + * inst_repos, e.g. 'el9'; defaults to null * next_version next daos package version to pass to daosPackagesVersion() if * testRpmArgs does not specify a daos_pkg_version; defaults to null * @return a scripted stage to run in a pipeline @@ -49,7 +51,7 @@ Map call(Map kwargs = [:]) { // Add defaults for any missing testRpm() arguments if (!testRpmArgs.containsKey('inst_repos')) { /* groovylint-disable-next-line DuplicateStringLiteral */ - testRpmArgs['inst_repos'] = daosRepos() + testRpmArgs['inst_repos'] = daosRepos(kwargs.get('distro', null)) } if (!testRpmArgs.containsKey('daos_pkg_version')) { /* groovylint-disable-next-line DuplicateStringLiteral */ From 9b496bbbc45af750c03a2b2303622fa817765fa5 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 15 Jul 2026 23:22:38 -0400 Subject: [PATCH 28/36] Debug Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/parseStageInfo.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vars/parseStageInfo.groovy b/vars/parseStageInfo.groovy index bf7719c60..00498f24e 100755 --- a/vars/parseStageInfo.groovy +++ b/vars/parseStageInfo.groovy @@ -57,9 +57,11 @@ Map call(Map config = [:]) { if (env.STAGE_NAME) { stage_name = env.STAGE_NAME } + println("parseStageInfo.DEBUG: stage_name: ${stage_name}") String new_ci_target = '' if (config['target']) { + println("parseStageInfo.DEBUG: config['target']: ${config['target']}") result['target'] = config['target'] } else if (env.TARGET) { result['target'] = env.TARGET @@ -72,10 +74,14 @@ Map call(Map config = [:]) { // Unified EL version handling for all major/minor releases } else if (stage_name.contains(' EL ')) { int elIdx = stage_name.indexOf('EL ') + println("parseStageInfo.DEBUG: elIdx: ${elIdx}") String elPart = stage_name.substring(elIdx + 3).split()[0] + println("parseStageInfo.DEBUG: elPart: ${elPart}") String[] parts = elPart.split('\\.') + println("parseStageInfo.DEBUG: parts: ${parts}") String majorVersion = parts[0] String minorVersion = parts.length > 1 ? parts[1] : null + println("parseStageInfo.DEBUG: majorVersion: ${majorVersion}, minorVersion: ${minorVersion}") if (minorVersion) { // Point release (e.g., EL 8.6, EL 9.4, EL 10.2) @@ -147,6 +153,7 @@ Map call(Map config = [:]) { } else { result['ci_target'] = result['target'] } + println("parseStageInfo.DEBUG: result['ci_target']: ${result['ci_target']}") if (result['ci_target'].startsWith('el') || result['ci_target'].startsWith('centos') || From 0c45b168c9806267e7b52f31e0898c40d2961e1c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 15 Jul 2026 23:59:45 -0400 Subject: [PATCH 29/36] More debug Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/parseStageInfo.groovy | 1 + vars/runTest.groovy | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/vars/parseStageInfo.groovy b/vars/parseStageInfo.groovy index 00498f24e..a5f0894e8 100755 --- a/vars/parseStageInfo.groovy +++ b/vars/parseStageInfo.groovy @@ -312,5 +312,6 @@ Map call(Map config = [:]) { } } + println("parseStageInfo.DEBUG: return ${result}") return result } diff --git a/vars/runTest.groovy b/vars/runTest.groovy index 71dcfad05..8050ffde4 100644 --- a/vars/runTest.groovy +++ b/vars/runTest.groovy @@ -65,6 +65,8 @@ Map call(Map config = [:]) { boolean ignore_failure = config.get('ignore_failure', false) boolean notify_result = config.get('notify_result', true) + println("runTest.DEBUG: Calling scmNotify w/ ${description}") + scmNotify description: description, context: context, status: 'PENDING' @@ -79,6 +81,7 @@ Map call(Map config = [:]) { config['failure_artifacts'] + '"' } + println("runTest.DEBUG: Running ${script}") String cb_result = currentBuild.result int rc = 255 try { @@ -135,6 +138,8 @@ Map call(Map config = [:]) { } } + println("runTest.DEBUG: Updating result ${status}") + long endDate = System.currentTimeMillis() int runTime = durationSeconds(startDate, endDate) From ffae8de729155bae5a127bb35d90de1fcc20bf4c Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 16 Jul 2026 00:48:06 -0400 Subject: [PATCH 30/36] Remove debug - problem was passing in NODELIST Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/parseStageInfo.groovy | 8 -------- vars/runTest.groovy | 5 ----- 2 files changed, 13 deletions(-) diff --git a/vars/parseStageInfo.groovy b/vars/parseStageInfo.groovy index a5f0894e8..bf7719c60 100755 --- a/vars/parseStageInfo.groovy +++ b/vars/parseStageInfo.groovy @@ -57,11 +57,9 @@ Map call(Map config = [:]) { if (env.STAGE_NAME) { stage_name = env.STAGE_NAME } - println("parseStageInfo.DEBUG: stage_name: ${stage_name}") String new_ci_target = '' if (config['target']) { - println("parseStageInfo.DEBUG: config['target']: ${config['target']}") result['target'] = config['target'] } else if (env.TARGET) { result['target'] = env.TARGET @@ -74,14 +72,10 @@ Map call(Map config = [:]) { // Unified EL version handling for all major/minor releases } else if (stage_name.contains(' EL ')) { int elIdx = stage_name.indexOf('EL ') - println("parseStageInfo.DEBUG: elIdx: ${elIdx}") String elPart = stage_name.substring(elIdx + 3).split()[0] - println("parseStageInfo.DEBUG: elPart: ${elPart}") String[] parts = elPart.split('\\.') - println("parseStageInfo.DEBUG: parts: ${parts}") String majorVersion = parts[0] String minorVersion = parts.length > 1 ? parts[1] : null - println("parseStageInfo.DEBUG: majorVersion: ${majorVersion}, minorVersion: ${minorVersion}") if (minorVersion) { // Point release (e.g., EL 8.6, EL 9.4, EL 10.2) @@ -153,7 +147,6 @@ Map call(Map config = [:]) { } else { result['ci_target'] = result['target'] } - println("parseStageInfo.DEBUG: result['ci_target']: ${result['ci_target']}") if (result['ci_target'].startsWith('el') || result['ci_target'].startsWith('centos') || @@ -312,6 +305,5 @@ Map call(Map config = [:]) { } } - println("parseStageInfo.DEBUG: return ${result}") return result } diff --git a/vars/runTest.groovy b/vars/runTest.groovy index 8050ffde4..71dcfad05 100644 --- a/vars/runTest.groovy +++ b/vars/runTest.groovy @@ -65,8 +65,6 @@ Map call(Map config = [:]) { boolean ignore_failure = config.get('ignore_failure', false) boolean notify_result = config.get('notify_result', true) - println("runTest.DEBUG: Calling scmNotify w/ ${description}") - scmNotify description: description, context: context, status: 'PENDING' @@ -81,7 +79,6 @@ Map call(Map config = [:]) { config['failure_artifacts'] + '"' } - println("runTest.DEBUG: Running ${script}") String cb_result = currentBuild.result int rc = 255 try { @@ -138,8 +135,6 @@ Map call(Map config = [:]) { } } - println("runTest.DEBUG: Updating result ${status}") - long endDate = System.currentTimeMillis() int runTime = durationSeconds(startDate, endDate) From ad7d41856540e3e3533d82d39a7598cf90f63b00 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 16 Jul 2026 09:53:01 -0400 Subject: [PATCH 31/36] Attempt to fix running testsInStage() Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/getFunctionalTestStage.groovy | 34 ++++++++++------------------- vars/skipFunctionalTestStage.groovy | 24 +++++++++++++++----- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/vars/getFunctionalTestStage.groovy b/vars/getFunctionalTestStage.groovy index 452bf952e..632fa8c03 100644 --- a/vars/getFunctionalTestStage.groovy +++ b/vars/getFunctionalTestStage.groovy @@ -1,4 +1,4 @@ -/* groovylint-disable NestedBlockDepth, VariableName */ +/* groovylint-disable NestedBlockDepth, VariableName, DuplicateStringLiteral */ // vars/getFunctionalTestStage.groovy import org.jenkinsci.plugins.pipeline.modeldefinition.Utils @@ -68,7 +68,6 @@ Map call(Map kwargs = [:]) { return { stage("${name}") { - /* groovylint-disable-next-line DuplicateStringLiteral */ if (runStage == 'false') { println("[${name}] Stage skipped by runStage=false") Utils.markStageSkippedForConditional("${name}") @@ -80,26 +79,17 @@ Map call(Map kwargs = [:]) { // 'fit' the cluster will be run. String tags = getFunctionalTags( pragma_suffix: pragma_suffix, stage_tags: stage_tags, default_tags: default_tags) - - // To be removed once all stages have been converted to use runStage. - /* groovylint-disable-next-line DuplicateStringLiteral */ - if (runStage == 'undefined') { - Map skip_kwargs = [ - 'tags': tags, - 'pragma_suffix': pragma_suffix, - 'distro': distro, - 'run_if_pr': run_if_pr, - 'run_if_landing': run_if_landing] - if (skipFunctionalTestStage(skip_kwargs)) { - println("[${name}] Stage skipped by skipFunctionalTestStage()") - Utils.markStageSkippedForConditional("${name}") - return - } + Map skipKwargs = ['tags': tags, 'basicCheck': false] + if (runStage == 'true') { + skipKwargs['basicCheck'] = true + } else { + skipKwargs['pragma_suffix'] = pragma_suffix + skipKwargs['distro'] = distro + skipKwargs['run_if_pr'] = run_if_pr + skipKwargs['run_if_landing'] = run_if_landing } - - // Verify tags are valid for the stage and that there are tests to run. - if (runStage == 'true' && !testsInStage(tags)) { - println("[${name}] Stage skipped by no tests matching the '${tags}' tags") + if (skipFunctionalTestStage(skipKwargs)) { + println("[${name}] Stage skipped by skipFunctionalTestStage()") Utils.markStageSkippedForConditional("${name}") return } @@ -136,7 +126,6 @@ Map call(Map kwargs = [:]) { provider: provider)['ftest_arg'], test_function: 'runTestFunctionalV2'] if (node_count != null) { - /* groovylint-disable-next-line DuplicateStringLiteral */ ftestConfig['node_count'] = node_count } jobStatusUpdate( @@ -157,7 +146,6 @@ Map call(Map kwargs = [:]) { /* groovylint-disable-next-line CatchException */ } catch (Exception finallyError) { println("[${name}] Caught exception in finally: ${finallyError}") - /* groovylint-disable-next-line DuplicateStringLiteral */ jobStatusUpdate(job_status, name, 'FAILURE') if (tryError == null) { /* groovylint-disable-next-line ThrowExceptionFromFinallyBlock */ diff --git a/vars/skipFunctionalTestStage.groovy b/vars/skipFunctionalTestStage.groovy index d99de55d3..865947d7f 100644 --- a/vars/skipFunctionalTestStage.groovy +++ b/vars/skipFunctionalTestStage.groovy @@ -12,6 +12,7 @@ * distro functional test stage distro (required for VM) * run_if_pr whether or not the stage should run for PR builds * run_if_landing whether or not the stage should run for landing builds + * basicCheck only verify if the stage has altready passed and has tests matching the tags * @return a String reason why the stage should be skipped; empty if the stage should run */ /* groovylint-disable-next-line MethodSize */ @@ -30,14 +31,19 @@ Map call(Map kwargs = [:]) { String build_param_value = paramsValue(build_param, '').toString() Boolean run_if_landing = kwargs['run_if_landing'] ?: false Boolean run_if_pr = kwargs['run_if_pr'] ?: false + Boolean basicCheck = kwargs.get('basicCheck', false) String target_branch = env.CHANGE_TARGET ? env.CHANGE_TARGET : env.BRANCH_NAME - println("[${env.STAGE_NAME}] Running skipFunctionalTestStage: " + - "tags=${tags}, pragma_suffix=${pragma_suffix}, size=${size}, distro=${distro}, " + - "build_param=${build_param}, build_param_value=${build_param_value}, " + - "run_if_landing=${run_if_landing}, run_if_pr=${run_if_pr}, " + - "startedByUser()=${startedByUser()}, startedByTimer()=${startedByTimer()}, " + - "startedByUpstream()=${startedByUpstream()}, target_branch=${target_branch}") + if (basicCheck) { + println("[${env.STAGE_NAME}] Running skipFunctionalTestStage: tags=${tags}") + } else { + println("[${env.STAGE_NAME}] Running skipFunctionalTestStage: " + + "tags=${tags}, pragma_suffix=${pragma_suffix}, size=${size}, distro=${distro}, " + + "build_param=${build_param}, build_param_value=${build_param_value}, " + + "run_if_landing=${run_if_landing}, run_if_pr=${run_if_pr}, " + + "startedByUser()=${startedByUser()}, startedByTimer()=${startedByTimer()}, " + + "startedByUpstream()=${startedByUpstream()}, target_branch=${target_branch}") + } // Regardless of how the stage has been started always skip a stage that has either already // passed or does not contain any tests match the tags. @@ -50,6 +56,12 @@ Map call(Map kwargs = [:]) { return true } + // With basicCheck set other stage skip criteria has been already confirmed externally + if (basicCheck) { + println("[${env.STAGE_NAME}] Stage has not already passed and has tests matching the tags.") + return false + } + // If the stage has been started by the user, e.g. Build with Parameters, or a timer, or an // upstream build then use the stage's build parameter (check box) to determine if the stage // should be run or skipped. From 8fa926acd61a3cbbb76d1bf792d478dafc095e8f Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 16 Jul 2026 17:21:52 -0400 Subject: [PATCH 32/36] Updates to testsInStage() Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/skipFunctionalTestStage.groovy | 2 +- vars/testsInStage.groovy | 57 +++++++++++++++-------------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/vars/skipFunctionalTestStage.groovy b/vars/skipFunctionalTestStage.groovy index 865947d7f..384901081 100644 --- a/vars/skipFunctionalTestStage.groovy +++ b/vars/skipFunctionalTestStage.groovy @@ -58,7 +58,7 @@ Map call(Map kwargs = [:]) { // With basicCheck set other stage skip criteria has been already confirmed externally if (basicCheck) { - println("[${env.STAGE_NAME}] Stage has not already passed and has tests matching the tags.") + println("[${env.STAGE_NAME}] Stage needs to run and has tests matching the tags.") return false } diff --git a/vars/testsInStage.groovy b/vars/testsInStage.groovy index 72305bae0..8daa933e3 100644 --- a/vars/testsInStage.groovy +++ b/vars/testsInStage.groovy @@ -1,4 +1,4 @@ -/* groovylint-disable DuplicateStringLiteral */ +/* groovylint-disable CatchException, DuplicateNumberLiteral, DuplicateStringLiteral */ // vars/testsInStage.groovy /** @@ -14,31 +14,32 @@ boolean call(String tags) { if (env.UNIT_TEST && env.UNIT_TEST == 'true') { return true } - /* groovylint-disable-next-line UnnecessaryGetter */ - String verbose = isPr() ? '--verbose ' : '' - return sh(label: 'Get test list', - /* groovylint-disable-next-line GStringExpressionWithinString */ - script: '''trap 'echo "Got an unhandled error, exiting as if a match was found"; exit 0' ERR - # This doesn't actually work on weekly-testing branches due to a lack - # src/test/ftest/launch.py (and friends). We could probably just - # check that out from the branch we are testing against (i.e. master, - # release/*, etc.) but let's save that for another day and just exit - # with a "tests found" for now. - if ! cd src/tests/ftest; then - echo "src/tests/ftest doesn't exist." - echo "Could not determine if tests exist for this stage, assuming they do." - exit 0 - fi - if [ -x list_tests.py ]; then - if ./list_tests.py ''' + tags + '''; then - exit 0 - fi - else - if ./launch.py --list ''' + verbose + tags + '''; then - exit 0 - fi - fi - trap '' ERR - exit 1''', - returnStatus: true) == 0 + + if (!fileExists('src/tests/ftest')) { + println("[${env.STAGE_NAME}] src/tests/ftest does not exist, assuming tests exist") + return true + } + try { + directory('src/tests/ftest') { + if (fileExists('list_tests.py')) { + return sh( + label: 'Run list_tests.py', + script: "./list_tests.py ${tags}", + returnStatus: true) == 0 + } + if (fileExists('launch.py')) { + /* groovylint-disable-next-line UnnecessaryGetter */ + String verbose = isPr() ? '--verbose ' : '' + return sh( + label: 'Run launch.py', + script: "./launch.py --list ${verbose} ${tags}", + returnStatus: true) == 0 + } + println("[${env.STAGE_NAME}] Neither list_tests.py or launch.py found") + } + } catch (Exception e) { + println("[${env.STAGE_NAME}] Caught exception in try: ${e}") + } + println("[${env.STAGE_NAME}] Could not determine if tests exist, assuming they do.") + return true } From 021404b0aee009b7a604de93b9426fc72952ee25 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 16 Jul 2026 18:01:31 -0400 Subject: [PATCH 33/36] Debug. Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/testsInStage.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vars/testsInStage.groovy b/vars/testsInStage.groovy index 8daa933e3..968088bcb 100644 --- a/vars/testsInStage.groovy +++ b/vars/testsInStage.groovy @@ -12,6 +12,7 @@ boolean call(String tags) { println("[${env.STAGE_NAME}] Determining if tests w/ '${tags}' tags exist for this stage") if (env.UNIT_TEST && env.UNIT_TEST == 'true') { + println("[${env.STAGE_NAME}] Ignoring the test existence check due to Unit Testing") return true } @@ -37,8 +38,8 @@ boolean call(String tags) { } println("[${env.STAGE_NAME}] Neither list_tests.py or launch.py found") } - } catch (Exception e) { - println("[${env.STAGE_NAME}] Caught exception in try: ${e}") + } catch (Exception error) { + println("[${env.STAGE_NAME}] Caught exception in try: ${error}") } println("[${env.STAGE_NAME}] Could not determine if tests exist, assuming they do.") return true From a5583e1d72f9a73747a45a173c546f4bd1c653f7 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 16 Jul 2026 23:02:49 -0400 Subject: [PATCH 34/36] Removing env.UNIT_TEST handling in testsInStage THe env.UNIT_TEST handling was bypassing the ability to check the tags for test matches in the stage. Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/testsInStage.groovy | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vars/testsInStage.groovy b/vars/testsInStage.groovy index 968088bcb..3347f86c5 100644 --- a/vars/testsInStage.groovy +++ b/vars/testsInStage.groovy @@ -11,11 +11,6 @@ */ boolean call(String tags) { println("[${env.STAGE_NAME}] Determining if tests w/ '${tags}' tags exist for this stage") - if (env.UNIT_TEST && env.UNIT_TEST == 'true') { - println("[${env.STAGE_NAME}] Ignoring the test existence check due to Unit Testing") - return true - } - if (!fileExists('src/tests/ftest')) { println("[${env.STAGE_NAME}] src/tests/ftest does not exist, assuming tests exist") return true From e38a6a523c8d47de2965c0d92ee56c71c28fe659 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 16 Jul 2026 23:39:46 -0400 Subject: [PATCH 35/36] Revert most of the testsInStage() changes Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- vars/testsInStage.groovy | 54 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/vars/testsInStage.groovy b/vars/testsInStage.groovy index 3347f86c5..b9c4965d1 100644 --- a/vars/testsInStage.groovy +++ b/vars/testsInStage.groovy @@ -11,31 +11,31 @@ */ boolean call(String tags) { println("[${env.STAGE_NAME}] Determining if tests w/ '${tags}' tags exist for this stage") - if (!fileExists('src/tests/ftest')) { - println("[${env.STAGE_NAME}] src/tests/ftest does not exist, assuming tests exist") - return true - } - try { - directory('src/tests/ftest') { - if (fileExists('list_tests.py')) { - return sh( - label: 'Run list_tests.py', - script: "./list_tests.py ${tags}", - returnStatus: true) == 0 - } - if (fileExists('launch.py')) { - /* groovylint-disable-next-line UnnecessaryGetter */ - String verbose = isPr() ? '--verbose ' : '' - return sh( - label: 'Run launch.py', - script: "./launch.py --list ${verbose} ${tags}", - returnStatus: true) == 0 - } - println("[${env.STAGE_NAME}] Neither list_tests.py or launch.py found") - } - } catch (Exception error) { - println("[${env.STAGE_NAME}] Caught exception in try: ${error}") - } - println("[${env.STAGE_NAME}] Could not determine if tests exist, assuming they do.") - return true + /* groovylint-disable-next-line UnnecessaryGetter */ + String verbose = isPr() ? '--verbose ' : '' + return sh(label: 'Get test list', + /* groovylint-disable-next-line GStringExpressionWithinString */ + script: '''trap 'echo "Got an unhandled error, exiting as if a match was found"; exit 0' ERR + # This doesn't actually work on weekly-testing branches due to a lack + # src/test/ftest/launch.py (and friends). We could probably just + # check that out from the branch we are testing against (i.e. master, + # release/*, etc.) but let's save that for another day and just exit + # with a "tests found" for now. + if ! cd src/tests/ftest; then + echo "src/tests/ftest doesn't exist." + echo "Could not determine if tests exist for this stage, assuming they do." + exit 0 + fi + if [ -x list_tests.py ]; then + if ./list_tests.py ''' + tags + '''; then + exit 0 + fi + else + if ./launch.py --list ''' + verbose + tags + '''; then + exit 0 + fi + fi + trap '' ERR + exit 1''', + returnStatus: true) == 0 } From db18bdc5635d7d6c3a4945603d3831228b0a11ae Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 16 Jul 2026 23:42:32 -0400 Subject: [PATCH 36/36] Add self-reference to verify unit tests Skip-daos-build-and-test: true Signed-off-by: Phil Henderson --- Jenkinsfile | 1 + vars/testsInStage.groovy | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 63791b616..4b961fada 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,6 +19,7 @@ // Then a second PR submitted to comment out the @Library line, and when it // is landed, both PR branches can be deleted. //@Library(value='pipeline-lib@my_branch_name') _ +@Library(value='pipeline-lib@hendersp/DAOS-19247-fix') _ /* groovylint-disable-next-line CompileStatic */ job_status_internal = [:] diff --git a/vars/testsInStage.groovy b/vars/testsInStage.groovy index b9c4965d1..690f1d0e5 100644 --- a/vars/testsInStage.groovy +++ b/vars/testsInStage.groovy @@ -1,4 +1,4 @@ -/* groovylint-disable CatchException, DuplicateNumberLiteral, DuplicateStringLiteral */ +/* groovylint-disable DuplicateStringLiteral */ // vars/testsInStage.groovy /**