diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index cff9a45b6e..c38a1db1af 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -352,46 +352,46 @@ def build(command, cell) { nodeExclusion = "&&!${NODE_NAME}" withEnv(cell.collect { k, v -> "${k}=${v}" }) { ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") { - fetchSource(cell.step, cell.arch, cell.jdk) - sh """ - test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; } - grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; } - """ - fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['debian-build']) - def cell_suffix = "_jdk${cell.jdk}_${cell.arch}" - def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz" - def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail - script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'" - timeout(time: 1, unit: 'HOURS') { - try { - def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true - dir("build") { - archiveArtifacts artifacts: "${logfile}", fingerprint: true - copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/") - } - if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } - if ("jar" == cell.step) { - stash name: "${cell.arch}_${cell.jdk}" - } - } catch (exc) { - if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) { - def descriptions = [] - for (def cause in exc.getCauses()) { - echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}" - if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) { - throw exc // user explicitly aborted — do not retry - } - descriptions.add(cause.getShortDescription()) + try { + fetchSource(cell.step, cell.arch, cell.jdk) + sh """ + test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; } + grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; } + """ + fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['debian-build']) + def cell_suffix = "_jdk${cell.jdk}_${cell.arch}" + def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz" + def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail + script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'" + timeout(time: 1, unit: 'HOURS') { + try { + def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true + dir("build") { + archiveArtifacts artifacts: "${logfile}", fingerprint: true + copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/") } - error("Retryable interruption: ${descriptions.join(', ')}") + if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } + if ("jar" == cell.step) { + stash name: "${cell.arch}_${cell.jdk}" + } + } catch (exc) { + if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) { + def descriptions = [] + for (def cause in exc.getCauses()) { + echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}" + if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) { + throw exc // user explicitly aborted — do not retry + } + descriptions.add(cause.getShortDescription()) + } + error("Retryable interruption: ${descriptions.join(', ')}") + } + throw exc } - throw exc } + } finally { + cleanAgent(cell.step) } - dir("build") { - copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/") - } - cleanAgent(cell.step) } } } @@ -410,69 +410,72 @@ def test(command, cell) { nodeExclusion = "&&!${NODE_NAME}" withEnv(cell.collect { k, v -> "${k}=${v}" }) { ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") { - fetchSource(cell.step, cell.arch, cell.jdk) - fetchDockerImages(['ubuntu-test']) - def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}" - def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz" - def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail - script_vars = "${script_vars} python_version=\'${cell.python}\'" - script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'" - if ("cqlsh-test" == cell.step) { - script_vars = "${script_vars} cython=\'${cell.cython}\'" - } - script_vars = fetchDTestsSource(command, script_vars) - timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes - def timer = System.currentTimeMillis() - try { - buildJVMDTestJars(cell, script_vars, logfile) - script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\"" - def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true - dir("build") { - archiveArtifacts artifacts: "${logfile}", fingerprint: true - } - if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } - } catch (exc) { - if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) { - def descriptions = [] - for (def cause in exc.getCauses()) { - echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}" - if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) { - throw exc // user explicitly aborted — do not retry - } - descriptions.add(cause.getShortDescription()) + try { + fetchSource(cell.step, cell.arch, cell.jdk) + fetchDockerImages(['ubuntu-test']) + def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}" + def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz" + def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail + script_vars = "${script_vars} python_version=\'${cell.python}\'" + script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'" + if ("cqlsh-test" == cell.step) { + script_vars = "${script_vars} cython=\'${cell.cython}\'" + } + script_vars = fetchDTestsSource(command, script_vars) + timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes + def timer = System.currentTimeMillis() + try { + buildJVMDTestJars(cell, script_vars, logfile) + script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\"" + def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true + dir("build") { + archiveArtifacts artifacts: "${logfile}", fingerprint: true } - error("Retryable interruption: ${descriptions.join(', ')}") + if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } + } catch (exc) { + if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) { + def descriptions = [] + for (def cause in exc.getCauses()) { + echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}" + if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) { + throw exc // user explicitly aborted — do not retry + } + descriptions.add(cause.getShortDescription()) + } + error("Retryable interruption: ${descriptions.join(', ')}") + } + throw exc + } finally { + def duration = System.currentTimeMillis() - timer + def formattedTime = String.format("%tT.%tL", duration, duration) + echo "Time ${cell.step}${cell_suffix}: ${formattedTime}" } - throw exc - } finally { - def duration = System.currentTimeMillis() - timer - def formattedTime = String.format("%tT.%tL", duration, duration) - echo "Time ${cell.step}${cell_suffix}: ${formattedTime}" } - } - dir("build") { - sh """ - mkdir -p test/output/${cell.step} - find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';' - find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';' - find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';' - """ - if (!cell.step.startsWith("microbench")) { - junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']] + dir("build") { + sh """ + mkdir -p test/output/${cell.step} + find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';' + find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';' + find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';' + """ + if (!cell.step.startsWith("microbench")) { + junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']] + } + // check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes + sh """ + echo "docker memory/oomkiller debug:" + cat /sys/fs/cgroup/docker/memory.events || true + """ + sh """ + find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f + echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l + """ + archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true + copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) } - // check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes - sh """ - echo "docker memory/oomkiller debug:" - cat /sys/fs/cgroup/docker/memory.events || true - """ - sh """ - find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f - echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l - """ - archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true - copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) + } finally { + cleanAgent(cell.step) } - cleanAgent(cell.step) } } } @@ -566,6 +569,13 @@ def cleanAgent(job_name) { logAgentInfo(job_name, agentScriptsUrl) } cleanWs() + if (isCanonical()) { + // in the workspace prune any abandoned or uncleaned builds (CASSANDRA-20436) + sh """#!/bin/bash + set +e + find /home/jenkins/jenkins-*/workspace/ -mindepth 2 -maxdepth 2 -type d -regextype posix-extended -regex '.*/[0-9]+' -mtime +31 -print -exec rm -rf {} + + """ + } } def cleanAgentDocker(job_name, agentScriptsUrl) {