mirror of https://github.com/apache/cassandra
Fix ci-cassandra.a.o agent workspaces for Cassandra-5.0 (and above) not being cleaned
The build+test cells run in `ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}")`, so every build writes a new per-build directory.
The `cleanWs()` only cleans the current cell's leaf, and it sat as the last statement in the `ws` block — so any cell hitting error() (failed/timed-out/aborted) exits before reaching it and leaves the full cell's workspace build behind.
patch by Mick Semb Wever; reviewed by Dmitry Konstantinov for CASSANDRA-20436
This commit is contained in:
mck
parent
b78f654fff
commit
26bdaa5ae5
|
|
@ -349,46 +349,49 @@ def build(command, cell) {
|
||||||
nodeExclusion = "&&!${NODE_NAME}"
|
nodeExclusion = "&&!${NODE_NAME}"
|
||||||
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
|
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
|
||||||
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
|
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
|
||||||
fetchSource(cell.step, cell.arch, cell.jdk)
|
try {
|
||||||
sh """
|
fetchSource(cell.step, cell.arch, cell.jdk)
|
||||||
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
|
sh """
|
||||||
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
|
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
|
||||||
"""
|
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
|
||||||
fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['bullseye-build'])
|
"""
|
||||||
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
|
fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['bullseye-build'])
|
||||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
|
||||||
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
|
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
||||||
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
|
||||||
timeout(time: 1, unit: 'HOURS') {
|
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
||||||
try {
|
timeout(time: 1, unit: 'HOURS') {
|
||||||
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
try {
|
||||||
dir("build") {
|
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
||||||
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
dir("build") {
|
||||||
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
||||||
}
|
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
||||||
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
|
||||||
if ("jar" == cell.step) {
|
|
||||||
stash name: "${cell.arch}_${cell.jdk}"
|
|
||||||
}
|
|
||||||
} catch (exc) {
|
|
||||||
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
|
|
||||||
def descriptions = []
|
|
||||||
for (def cause in exc.getCauses()) {
|
|
||||||
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
|
|
||||||
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
|
|
||||||
throw exc // user explicitly aborted — do not retry
|
|
||||||
}
|
|
||||||
descriptions.add(cause.getShortDescription())
|
|
||||||
}
|
}
|
||||||
error("Retryable interruption: ${descriptions.join(', ')}")
|
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
||||||
|
if ("jar" == cell.step) {
|
||||||
|
stash name: "${cell.arch}_${cell.jdk}"
|
||||||
|
}
|
||||||
|
dir("build") {
|
||||||
|
copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
||||||
|
}
|
||||||
|
} catch (exc) {
|
||||||
|
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
|
||||||
|
def descriptions = []
|
||||||
|
for (def cause in exc.getCauses()) {
|
||||||
|
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
|
||||||
|
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
|
||||||
|
throw exc // user explicitly aborted — do not retry
|
||||||
|
}
|
||||||
|
descriptions.add(cause.getShortDescription())
|
||||||
|
}
|
||||||
|
error("Retryable interruption: ${descriptions.join(', ')}")
|
||||||
|
}
|
||||||
|
throw exc
|
||||||
}
|
}
|
||||||
throw exc
|
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
cleanAgent(cell.step)
|
||||||
}
|
}
|
||||||
dir("build") {
|
|
||||||
copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
|
||||||
}
|
|
||||||
cleanAgent(cell.step)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -407,69 +410,72 @@ def test(command, cell) {
|
||||||
nodeExclusion = "&&!${NODE_NAME}"
|
nodeExclusion = "&&!${NODE_NAME}"
|
||||||
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
|
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
|
||||||
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
|
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
|
||||||
fetchSource(cell.step, cell.arch, cell.jdk)
|
try {
|
||||||
fetchDockerImages(['ubuntu-test'])
|
fetchSource(cell.step, cell.arch, cell.jdk)
|
||||||
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
|
fetchDockerImages(['ubuntu-test'])
|
||||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
|
||||||
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
|
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
||||||
script_vars = "${script_vars} python_version=\'${cell.python}\'"
|
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
|
||||||
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
script_vars = "${script_vars} python_version=\'${cell.python}\'"
|
||||||
if ("cqlsh-test" == cell.step) {
|
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
||||||
script_vars = "${script_vars} cython=\'${cell.cython}\'"
|
if ("cqlsh-test" == cell.step) {
|
||||||
}
|
script_vars = "${script_vars} cython=\'${cell.cython}\'"
|
||||||
script_vars = fetchDTestsSource(command, script_vars)
|
}
|
||||||
timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
|
script_vars = fetchDTestsSource(command, script_vars)
|
||||||
def timer = System.currentTimeMillis()
|
timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
|
||||||
try {
|
def timer = System.currentTimeMillis()
|
||||||
buildJVMDTestJars(cell, script_vars, logfile)
|
try {
|
||||||
script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\""
|
buildJVMDTestJars(cell, script_vars, logfile)
|
||||||
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\""
|
||||||
dir("build") {
|
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
||||||
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
dir("build") {
|
||||||
}
|
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
||||||
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
|
||||||
} catch (exc) {
|
|
||||||
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
|
|
||||||
def descriptions = []
|
|
||||||
for (def cause in exc.getCauses()) {
|
|
||||||
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
|
|
||||||
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
|
|
||||||
throw exc // user explicitly aborted — do not retry
|
|
||||||
}
|
|
||||||
descriptions.add(cause.getShortDescription())
|
|
||||||
}
|
}
|
||||||
error("Retryable interruption: ${descriptions.join(', ')}")
|
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
||||||
|
} catch (exc) {
|
||||||
|
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
|
||||||
|
def descriptions = []
|
||||||
|
for (def cause in exc.getCauses()) {
|
||||||
|
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
|
||||||
|
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
|
||||||
|
throw exc // user explicitly aborted — do not retry
|
||||||
|
}
|
||||||
|
descriptions.add(cause.getShortDescription())
|
||||||
|
}
|
||||||
|
error("Retryable interruption: ${descriptions.join(', ')}")
|
||||||
|
}
|
||||||
|
throw exc
|
||||||
|
} finally {
|
||||||
|
def duration = System.currentTimeMillis() - timer
|
||||||
|
def formattedTime = String.format("%tT.%tL", duration, duration)
|
||||||
|
echo "Time ${cell.step}${cell_suffix}: ${formattedTime}"
|
||||||
}
|
}
|
||||||
throw exc
|
|
||||||
} finally {
|
|
||||||
def duration = System.currentTimeMillis() - timer
|
|
||||||
def formattedTime = String.format("%tT.%tL", duration, duration)
|
|
||||||
echo "Time ${cell.step}${cell_suffix}: ${formattedTime}"
|
|
||||||
}
|
}
|
||||||
}
|
dir("build") {
|
||||||
dir("build") {
|
sh """
|
||||||
sh """
|
mkdir -p test/output/${cell.step}
|
||||||
mkdir -p test/output/${cell.step}
|
find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
|
||||||
find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
|
find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
|
||||||
find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
|
find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
|
||||||
find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
|
"""
|
||||||
"""
|
if (!cell.step.startsWith("microbench")) {
|
||||||
if (!cell.step.startsWith("microbench")) {
|
junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
|
||||||
junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
|
}
|
||||||
|
// check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
|
||||||
|
sh """
|
||||||
|
echo "docker memory/oomkiller debug:"
|
||||||
|
cat /sys/fs/cgroup/docker/memory.events || true
|
||||||
|
"""
|
||||||
|
sh """
|
||||||
|
find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
|
||||||
|
echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
|
||||||
|
"""
|
||||||
|
archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
|
||||||
|
copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
|
||||||
}
|
}
|
||||||
// check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
|
} finally {
|
||||||
sh """
|
cleanAgent(cell.step)
|
||||||
echo "docker memory/oomkiller debug:"
|
|
||||||
cat /sys/fs/cgroup/docker/memory.events || true
|
|
||||||
"""
|
|
||||||
sh """
|
|
||||||
find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
|
|
||||||
echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
|
|
||||||
"""
|
|
||||||
archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
|
|
||||||
copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
|
|
||||||
}
|
}
|
||||||
cleanAgent(cell.step)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -563,6 +569,13 @@ def cleanAgent(job_name) {
|
||||||
logAgentInfo(job_name, agentScriptsUrl)
|
logAgentInfo(job_name, agentScriptsUrl)
|
||||||
}
|
}
|
||||||
cleanWs()
|
cleanWs()
|
||||||
|
if (isCanonical()) {
|
||||||
|
// in the workspace prune any abandoned or uncleaned builds (CASSANDRA-20436)
|
||||||
|
sh """#!/bin/bash
|
||||||
|
set +e
|
||||||
|
find /home/jenkins/jenkins-*/workspace/ -mindepth 2 -maxdepth 2 -type d -regextype posix-extended -regex '.*/[0-9]+' -mtime +31 -print -exec rm -rf {} +
|
||||||
|
"""
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def cleanAgentDocker(job_name, agentScriptsUrl) {
|
def cleanAgentDocker(job_name, agentScriptsUrl) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue