mirror of https://github.com/apache/cassandra
Fix ci-cassandra.a.o agent workspaces for Cassandra-5.0 (and above) not being cleaned
The build+test cells run in `ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}")`, so every build writes a new per-build directory.
The `cleanWs()` only cleans the current cell's leaf, and it sat as the last statement in the `ws` block — so any cell hitting error() (failed/timed-out/aborted) exits before reaching it and leaves the full cell's workspace build behind.
patch by Mick Semb Wever; reviewed by Dmitry Konstantinov for CASSANDRA-20436
This commit is contained in:
mck
parent
b78f654fff
commit
26bdaa5ae5
|
|
@ -349,46 +349,49 @@ def build(command, cell) {
|
|||
nodeExclusion = "&&!${NODE_NAME}"
|
||||
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
|
||||
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
|
||||
fetchSource(cell.step, cell.arch, cell.jdk)
|
||||
sh """
|
||||
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
|
||||
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
|
||||
"""
|
||||
fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['bullseye-build'])
|
||||
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
|
||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
||||
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
|
||||
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
||||
timeout(time: 1, unit: 'HOURS') {
|
||||
try {
|
||||
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
||||
dir("build") {
|
||||
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
||||
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
||||
}
|
||||
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
||||
if ("jar" == cell.step) {
|
||||
stash name: "${cell.arch}_${cell.jdk}"
|
||||
}
|
||||
} catch (exc) {
|
||||
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
|
||||
def descriptions = []
|
||||
for (def cause in exc.getCauses()) {
|
||||
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
|
||||
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
|
||||
throw exc // user explicitly aborted — do not retry
|
||||
}
|
||||
descriptions.add(cause.getShortDescription())
|
||||
try {
|
||||
fetchSource(cell.step, cell.arch, cell.jdk)
|
||||
sh """
|
||||
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
|
||||
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
|
||||
"""
|
||||
fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['bullseye-build'])
|
||||
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
|
||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
||||
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
|
||||
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
||||
timeout(time: 1, unit: 'HOURS') {
|
||||
try {
|
||||
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
||||
dir("build") {
|
||||
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
||||
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
||||
}
|
||||
error("Retryable interruption: ${descriptions.join(', ')}")
|
||||
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
||||
if ("jar" == cell.step) {
|
||||
stash name: "${cell.arch}_${cell.jdk}"
|
||||
}
|
||||
dir("build") {
|
||||
copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
||||
}
|
||||
} catch (exc) {
|
||||
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
|
||||
def descriptions = []
|
||||
for (def cause in exc.getCauses()) {
|
||||
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
|
||||
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
|
||||
throw exc // user explicitly aborted — do not retry
|
||||
}
|
||||
descriptions.add(cause.getShortDescription())
|
||||
}
|
||||
error("Retryable interruption: ${descriptions.join(', ')}")
|
||||
}
|
||||
throw exc
|
||||
}
|
||||
throw exc
|
||||
}
|
||||
} finally {
|
||||
cleanAgent(cell.step)
|
||||
}
|
||||
dir("build") {
|
||||
copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
||||
}
|
||||
cleanAgent(cell.step)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -407,69 +410,72 @@ def test(command, cell) {
|
|||
nodeExclusion = "&&!${NODE_NAME}"
|
||||
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
|
||||
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
|
||||
fetchSource(cell.step, cell.arch, cell.jdk)
|
||||
fetchDockerImages(['ubuntu-test'])
|
||||
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
|
||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
||||
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
|
||||
script_vars = "${script_vars} python_version=\'${cell.python}\'"
|
||||
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
||||
if ("cqlsh-test" == cell.step) {
|
||||
script_vars = "${script_vars} cython=\'${cell.cython}\'"
|
||||
}
|
||||
script_vars = fetchDTestsSource(command, script_vars)
|
||||
timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
|
||||
def timer = System.currentTimeMillis()
|
||||
try {
|
||||
buildJVMDTestJars(cell, script_vars, logfile)
|
||||
script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\""
|
||||
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
||||
dir("build") {
|
||||
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
||||
}
|
||||
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
||||
} catch (exc) {
|
||||
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
|
||||
def descriptions = []
|
||||
for (def cause in exc.getCauses()) {
|
||||
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
|
||||
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
|
||||
throw exc // user explicitly aborted — do not retry
|
||||
}
|
||||
descriptions.add(cause.getShortDescription())
|
||||
try {
|
||||
fetchSource(cell.step, cell.arch, cell.jdk)
|
||||
fetchDockerImages(['ubuntu-test'])
|
||||
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
|
||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
||||
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
|
||||
script_vars = "${script_vars} python_version=\'${cell.python}\'"
|
||||
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
||||
if ("cqlsh-test" == cell.step) {
|
||||
script_vars = "${script_vars} cython=\'${cell.cython}\'"
|
||||
}
|
||||
script_vars = fetchDTestsSource(command, script_vars)
|
||||
timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
|
||||
def timer = System.currentTimeMillis()
|
||||
try {
|
||||
buildJVMDTestJars(cell, script_vars, logfile)
|
||||
script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\""
|
||||
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
||||
dir("build") {
|
||||
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
||||
}
|
||||
error("Retryable interruption: ${descriptions.join(', ')}")
|
||||
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
||||
} catch (exc) {
|
||||
if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
|
||||
def descriptions = []
|
||||
for (def cause in exc.getCauses()) {
|
||||
echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
|
||||
if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
|
||||
throw exc // user explicitly aborted — do not retry
|
||||
}
|
||||
descriptions.add(cause.getShortDescription())
|
||||
}
|
||||
error("Retryable interruption: ${descriptions.join(', ')}")
|
||||
}
|
||||
throw exc
|
||||
} finally {
|
||||
def duration = System.currentTimeMillis() - timer
|
||||
def formattedTime = String.format("%tT.%tL", duration, duration)
|
||||
echo "Time ${cell.step}${cell_suffix}: ${formattedTime}"
|
||||
}
|
||||
throw exc
|
||||
} finally {
|
||||
def duration = System.currentTimeMillis() - timer
|
||||
def formattedTime = String.format("%tT.%tL", duration, duration)
|
||||
echo "Time ${cell.step}${cell_suffix}: ${formattedTime}"
|
||||
}
|
||||
}
|
||||
dir("build") {
|
||||
sh """
|
||||
mkdir -p test/output/${cell.step}
|
||||
find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
|
||||
find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
|
||||
find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
|
||||
"""
|
||||
if (!cell.step.startsWith("microbench")) {
|
||||
junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
|
||||
dir("build") {
|
||||
sh """
|
||||
mkdir -p test/output/${cell.step}
|
||||
find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
|
||||
find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
|
||||
find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
|
||||
"""
|
||||
if (!cell.step.startsWith("microbench")) {
|
||||
junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
|
||||
}
|
||||
// check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
|
||||
sh """
|
||||
echo "docker memory/oomkiller debug:"
|
||||
cat /sys/fs/cgroup/docker/memory.events || true
|
||||
"""
|
||||
sh """
|
||||
find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
|
||||
echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
|
||||
"""
|
||||
archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
|
||||
copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
|
||||
}
|
||||
// check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
|
||||
sh """
|
||||
echo "docker memory/oomkiller debug:"
|
||||
cat /sys/fs/cgroup/docker/memory.events || true
|
||||
"""
|
||||
sh """
|
||||
find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
|
||||
echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
|
||||
"""
|
||||
archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
|
||||
copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
|
||||
} finally {
|
||||
cleanAgent(cell.step)
|
||||
}
|
||||
cleanAgent(cell.step)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -563,6 +569,13 @@ def cleanAgent(job_name) {
|
|||
logAgentInfo(job_name, agentScriptsUrl)
|
||||
}
|
||||
cleanWs()
|
||||
if (isCanonical()) {
|
||||
// in the workspace prune any abandoned or uncleaned builds (CASSANDRA-20436)
|
||||
sh """#!/bin/bash
|
||||
set +e
|
||||
find /home/jenkins/jenkins-*/workspace/ -mindepth 2 -maxdepth 2 -type d -regextype posix-extended -regex '.*/[0-9]+' -mtime +31 -print -exec rm -rf {} +
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
def cleanAgentDocker(job_name, agentScriptsUrl) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue