DAOS-19247 test: Support scripted Test stages

vars/getFunctionalTestStage.groovy

@@ -135,7 +135,7 @@ Map call(Map kwargs = [:]) {
                     jobStatusUpdate(job_status, name)
                 }
             }
+            println("[${name}] Finished with ${job_status}")
         }
-        println("[${name}] Finished with ${job_status}")
     }
 }

vars/scriptedDockerStage.groovy

@@ -0,0 +1,142 @@
+// vars/scriptedDockerStage.groovy
+
+import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
+
+/**
+ * scriptedDockerStage
+ *
+ * Get a build stage in scripted syntax.
+ *
+ * @param kwargs Map containing the following optional arguments (empty strings yield defaults):
+ *          name                    the build stage name
+ *          runStage                optional additional condition to determine if the stage runs
+ *          jobStatus               Map of status for each stage in the job/build
+ *          dockerTag               the docker image tag to use for the build
+ *          dockerBuildArgs         optional docker build arguments
+ *          buildRpms               whether or not to build rpms; defaults to true
+ *          distro                  the shorthand distro name; defaults to 'el8'
+ *          rpmDistro               the distro to use for rpm building; defaults to distro
+ *          release                 the DAOS RPM release value to use; defaults to env.DAOS_RELVAL
+ *          compiler                the compiler to use; defaults to 'gcc'
+ *          sconsBuildArgs          optional scons build arguments
+ *          valgrindSconsBuildArgs  optional scons build arguments for valgrind build
+ *          artifacts               optional artifacts name to archive; defaults to
+ *                                     "config.log-${distro}-${compiler}"
+ *          uploadTarget            the distro to use when uploading rpms; defaults to distro
+ *          installScript           optional script to run to install rpms; defaults to ''
+ *          publishHtmlArgs         optional arguments to pass to publishHTML()
+ *          archiveArtifactsArgs    optional arguments to pass to archiveArtifacts()
+ * @return a scripted stage to run in a pipeline
+ */
+Map call(Map kwargs = [:]) {
+    String name = kwargs.get('name', 'Unknown Docker Stage')
+    Boolean runStage = kwargs.get('runStage', true)
+    Map jobStatus = kwargs.get('jobStatus', [:])
+    String dockerTag = jobStatusKey("build-${uploadTarget}-${compiler}").toLowerCase()
+    String dockerBuildArgs = kwargs.get('dockerBuildArgs', '')
+
+    // Build stage parameters
+    Boolean buildRpms = kwargs.get('buildRpms', true)
+    String distro = kwargs.get('distro', 'el8')
+    String rpmDistro = kwargs.get('rpmDistro', distro)
+    String release = kwargs.get('release', env.DAOS_RELVAL)
+    String compiler = kwargs.get('compiler', 'gcc')
+    Map sconsBuildArgs = kwargs.get('sconsBuildArgs', [:])
+    Map valgrindSconsBuildArgs = kwargs.get('valgrindSconsBuildArgs', [:])
+    String artifacts = kwargs.get('artifacts', "config.log-${distro}-${compiler}")
+    String uploadTarget = kwargs.get('uploadTarget', distro)
+    String bullseye = 'false'
+    if (compiler == 'covc') {
+        bullseye = 'true'
+    }
+
+    // Summary stage parameters
+    String installScript = kwargs.get('installScript', '')
+    Map publishHtmlArgs = kwargs.get('publishHtmlArgs', [:])
+    Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', [:])
+
+    return {
+        stage("${name}") {
+            if (!runStage) {
+                println("[${name}] Marking docker stage as skipped")
+                Utils.markStageSkippedForConditional("${name}")
+                return
+            }
+            node('docker_runner') {
+                println("[${name}] Check out from version control")
+                checkoutScm(pruneStaleBranch: true)
+
+                def dockerImage = docker.build(dockerTag, dockerBuildArgs)
+                try {
+                    dockerImage.inside() {
+                        if (buildRpms) {
+                            sh label: 'Install RPMs',
+                                script: "./ci/rpm/install_deps.sh ${rpmDistro} ${release} ${bullseye}"
+                            // Avoid interpolation of sensitive environment variables
+                            sh label: 'Build deps',
+                                script: "./ci/rpm/build_deps.sh ${bullseye}" + ' ${BULLSEYE_KEY}'
+                        }
+                        if (sconsBuildArgs) {
+                            job_step_update(sconsBuild(sconsBuildArgs))
+                        }
+                        if (valgrindSconsBuildArgs) {
+                            // For non-release builds, create a separate build with the valgrind
+                            // tag for NLT memcheck testing.  This is necessary to avoid problems
+                            // caused by valgrind being confused by the Go runtime. We don't want
+                            // to use the valgrind build for normal testing because it is much
+                            // slower. BUILD_TYPE=dev is set for PR/dev builds in sconsArgs(), and
+                            // TARGET_TYPE=release is used to select pre-built cached prerequisites.
+                            job_step_update(sconsBuild(valgrindSconsBuildArgs))
+                            sh label: 'Stash valgrind install tree for NLT',
+                               script: 'tar -C / -cf opt-daos-valgrind.tar opt/daos'
+                            stash(name: 'opt-daos-valgrind', includes: 'opt-daos-valgrind.tar')
+                        }
+                        if (buildRpms) {
+                            sh label: 'Generate RPMs',
+                                script: "./ci/rpm/gen_rpms.sh ${rpmDistro} ${release} ${bullseye}"
+                            // Success actions
+                            uploadNewRPMs(uploadTarget, 'success')
+                        }
+                        if (installScript) {
+                            sh label: 'Install RPMs',
+                                script: "${installScript} ${distro}"
+                        }
+                    } 
+                } catch (Exception e) {
+                    println("[${name}] Caught exception in try: ${e}")
+                    if (sconsBuildArgs) {
+                        // Unsuccessful actions
+                        sh """if [ -f config.log ]; then
+                                mv config.log ${artifacts}
+                            fi"""
+                        archiveArtifacts artifacts: "${artifacts}", allowEmptyArchive: true
+                    }
+                    jobStatusUpdate(jobStatus, name, 'FAILURE')
+                    throw e
+                } finally {
+                    // Cleanup actions
+                    try{
+                        if (buildRpms) {
+                            println("[${name}] Running uploadNewRPMs()")
+                            uploadNewRPMs(uploadTarget, 'cleanup')
+                        }
+                        if (publishHtmlArgs) {
+                            println("[${name}] Running publishHTML()")
+                            publishHTML(publishHtmlArgs)
+                        }
+                        if (archiveArtifactsArgs) {
+                            println("[${name}] Running archiveArtifacts()")
+                            archiveArtifacts(archiveArtifactsArgs)
+                        }
+                        jobStatusUpdate(jobStatus, name)
+                    } catch (Exception e) {
+                        println("[${name}] Caught exception in finally: ${e}")
+                        jobStatusUpdate(jobStatus, name, 'FAILURE')
+                        throw e
+                    }
+                }
+            }
+            println("[${name}] Finished with ${jobStatus}")
+        }
+    }
+}

vars/scriptedTestRpmStage.groovy

@@ -0,0 +1,95 @@
+// vars/scriptedTestRpmStage.groovy
+
+import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
+
+/**
+ * scriptedTestRpmStage.groovy
+ *
+ * Get a test stage in scripted syntax.
+ *
+ * @param kwargs Map containing the following optional arguments (empty strings yield defaults):
+ *      name                  test stage name
+ *      runStage              whether or not to run the test stage
+ *      label                 test stage default cluster label
+ *      testBranch            if specified, checkout sources from this branch before running tests
+ *      jobStatus             Map of status for each stage in the job/build
+ *      imageVersion          testRpm() target argument; defaults to ''
+ *      instRepos             testRpm() inst_repos argument; defaults to daosRepos()
+ *      daosPkgVersion        testRpm() daos_pkg_version argument; defaults to
+ *                              daosPackagesVersion(next_version())
+ *      instRpms              testRpm() inst_rpms argument; defaults to 'mercury-libfabric'
+ *      ignoreFailure         testRpm() ignore_failure argument; defaults to false
+ *      alwaysScript          script to run always after the test stage; defaults to ''.
+ *      archiveArtifactsArgs  Map of arguments to pass to archiveArtifacts() for the stage
+ * @return a scripted stage to run in a pipeline
+ */ 
+Map call(Map kwargs = [:]) {
+    // General parameters
+    String name = kwargs.get('name', 'Unknown Functional Test Stage')
+    Boolean runStage = kwargs.get('runStage', true) as Boolean
+    String label = kwargs.get('label')
+    String testBranch = kwargs.get('testBranch', '')
+    Map jobStatus = kwargs.get('jobStatus', null) ?: [:]
+    Map testRpmArgs = [
+        target: kwargs.get('imageVersion', ''),
+        inst_repos: kwargs.get('instRepos', daosRepos()),
+        daos_pkg_version: kwargs.get(
+            'daosPkgVersion', daosPackagesVersion(kwargs.get('next_version', null))),
+        inst_rpms: kwargs.get('instRpms', 'mercury-libfabric'),
+        ignore_failure: kwargs.get('ignoreFailure', false)
+    ]
+    String alwaysScript = kwargs.get('alwaysScript', '')
+    Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:]
+
+    return {
+        stage("${name}") {
+            println("[${name}] Starting stage: kwargs keys=${kwargs.keySet()}")
+
+            if (!runStage) {
+                println("[${name}] Stage skipped by runStage=false")
+                Utils.markStageSkippedForConditional("${name}")
+                return
+            }
+
+            node(label) {
+                // Ensure access to any branch provisioning scripts exist
+                if (testBranch) {
+                    println("[${name}] Check out '${testBranch}' from version control")
+                    checkoutScm(
+                        url: 'https://github.com/daos-stack/daos.git',
+                        branch: testBranch,
+                        withSubmodules: false,
+                        pruneStaleBranch: true)
+                } else {
+                    println("[${name}] Check out branch from version control")
+                    checkoutScm(pruneStaleBranch: true)
+                }
+
+                try {
+                    println("[${name}] Running testRpm() on ${label}")
+                    jobStatusUpdate(jobStatus, name, testRpm(testRpmArgs))
+                } catch (Exception e) {
+                    println("[${name}] Caught exception in try: ${e}")
+                    jobStatusUpdate(jobStatus, name, 'FAILURE')
+                    throw e
+                } finally {
+                    try {
+                        if (alwaysScript) {
+                            sh(script: alwaysScript, label: "Running alwaysScript: ${alwaysScript}")
+                        }
+                        if (archiveArtifactsArgs) {
+                            println("[${name}] Running archiveArtifacts()")
+                            archiveArtifacts(archiveArtifactsArgs)
+                        }
+                        jobStatusUpdate(jobStatus, name)
+                    } catch (Exception e) {
+                        println("[${name}] Caught exception in finally: ${e}")
+                        jobStatusUpdate(jobStatus, name, 'FAILURE')
+                        throw e
+                    }
+                }
+            }
+            println("[${name}] Finished with ${jobStatus}")
+        }
+    }
+}

vars/scriptedUnitTestStage.groovy

@@ -0,0 +1,110 @@
+// vars/scriptedUnitTestStage.groovy
+
+import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
+
+/**
+ * scriptedUnitTestStage.groovy
+ *
+ * Get a unit test stage in scripted syntax.
+ *
+ * @param kwargs Map containing the following optional arguments (empty strings yield defaults):
+ *      name                  test stage name
+ *      runStage              whether or not to run the test stage
+ *      label                 test stage default cluster label
+ *      testBranch            if specified, checkout sources from this branch before running tests
+ *      jobStatus             Map of status for each stage in the job/build
+ *      target                the target distro to use for the test stage; defaults to 'el9'
+ *      timeoutTime           unitTest() timeout time in minutes; defaults to 60
+ *      instRepos             unitTest() inst_repos argument; defaults to daosRepos()
+ *      testScript            unitTest() test_script argument; defaults to ''
+ *      withValgrind          unitTest() with_valgrind argument; defaults to ''
+ *      alwaysScript          unitTest() always_script argument; defaults to 'ci/unit/test_nlt_post.sh'
+ *      testResults           unitTest() test_results argument; defaults to 'nlt-junit.xml'
+ *      unstashOpt            unitTest() unstash_opt argument; defaults to true
+ *      unstashTests          unitTest() unstash_tests argument; defaults to false
+ *      instRpms              unitTest() inst_rpms argument; defaults to unitPackages(target: target) + ' daos-client-tests'
+ *      imageVersion          unitTest() image_version argument; defaults to 'el9.7'
+ *      provEnvVars           unitTest() prov_env_vars argument; defaults to ''
+ *      unitTestPostArgs      Map of arguments to pass to unitTestPost() for the stage
+ *      archiveArtifactsArgs  Map of arguments to pass to archiveArtifacts() for the stage
+ * @return a scripted stage to run in a pipeline
+ */ 
+Map call(Map kwargs = [:]) {
+    // General parameters
+    String name = kwargs.get('name', 'Unknown Functional Test Stage')
+    Boolean runStage = kwargs.get('runStage', true) as Boolean
+    String label = kwargs.get('label')
+    String testBranch = kwargs.get('testBranch', '')
+    Map jobStatus = kwargs.get('jobStatus', null) ?: [:]
+    String distro = kwargs.get('distro', 'el9')
+
+    // Unit Test stage parameters
+    Map unitTestArgs = [
+        'timeout_time': kwargs.get('timeoutTime', 60),
+        'inst_repos': kwargs.get('instRepos', daosRepos(distro)),
+        'test_script': kwargs.get('testScript', ''),
+        'with_valgrind': kwargs.get('withValgrind', ''),
+        'always_script': kwargs.get('alwaysScript', 'ci/unit/test_nlt_post.sh'),
+        'test_results': kwargs.get('testResults', 'nlt-junit.xml'),
+        'unstash_opt': kwargs.get('unstashOpt', true),
+        'unstash_tests': kwargs.get('unstashTests', false),
+        'inst_rpms': kwargs.get('instRpms', unitPackages(target: distro) + ' daos-client-tests'),
+        'image_version': kwargs.get('imageVersion', 'el9.7'),
+        'prov_env_vars': kwargs.get('provEnvVars', '')
+    ]
+    Map unitTestPostArgs = kwargs.get('unitTestPostArgs', null) ?: [:]
+    Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', null) ?: [:]
+
+    return {
+        stage("${name}") {
+            println("[${name}] Starting stage: kwargs keys=${kwargs.keySet()}")
+
+            if (!runStage) {
+                println("[${name}] Stage skipped by runStage=false")
+                Utils.markStageSkippedForConditional("${name}")
+                return
+            }
+
+            node(label) {
+                // Ensure access to any branch provisioning scripts exist
+                if (testBranch) {
+                    println("[${name}] Check out '${testBranch}' from version control")
+                    checkoutScm(
+                        url: 'https://github.com/daos-stack/daos.git',
+                        branch: testBranch,
+                        withSubmodules: false,
+                        pruneStaleBranch: true)
+                } else {
+                    println("[${name}] Check out branch from version control")
+                    checkoutScm(pruneStaleBranch: true)
+                }
+
+                try {
+                    println("[${name}] Running unitTest() on ${label}")
+                    jobStatusUpdate(jobStatus, name, unitTest(unitTestArgs))
+                } catch (Exception e) {
+                    println("[${name}] Caught exception in try: ${e}")
+                    jobStatusUpdate(jobStatus, name, 'FAILURE')
+                    throw e
+                } finally {
+                    try {
+                        if (unitTestPostArgs) {
+                            println("[${name}] Running unitTestPost()")
+                            unitTestPost(unitTestPostArgs)
+                        }
+                        if (archiveArtifactsArgs) {
+                            println("[${name}] Running archiveArtifacts()")
+                            archiveArtifacts(archiveArtifactsArgs)
+                        }
+                        jobStatusUpdate(jobStatus, name)
+                    } catch (Exception e) {
+                        println("[${name}] Caught exception in finally: ${e}")
+                        jobStatusUpdate(jobStatus, name, 'FAILURE')
+                        throw e
+                    }
+                }
+            }
+            println("[${name}] Finished with ${jobStatus}")
+        }
+    }
+}