From 461b8c42d24b6906332949fa6f1bf110d08b7f06 Mon Sep 17 00:00:00 2001 From: Mick Semb Wever Date: Sat, 8 Jun 2024 15:00:04 +0200 Subject: [PATCH] Remove sid source from debian, was only needed for jdk8 - Added debug in testing - Propagate the $DEBUG env from docker to test scripts - Be verbose when a pipeline errors that not all tests would have been run - add timeout to each cell stage (scripts on a node can hang and the only alternative was to abort the whole pipeline) patch by Mick Semb Wever; reviewed by Ekaterina Dimitrova, Brandon Williams for CASSANDRA-19708 --- .build/docker/bullseye-build.docker | 2 -- .build/docker/run-tests.sh | 1 + .build/run-python-dtests.sh | 1 + .build/run-tests.sh | 2 ++ .jenkins/Jenkinsfile | 35 +++++++++++++++++------------ 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.build/docker/bullseye-build.docker b/.build/docker/bullseye-build.docker index c6c2928dc4..6928ec9f29 100644 --- a/.build/docker/bullseye-build.docker +++ b/.build/docker/bullseye-build.docker @@ -41,8 +41,6 @@ RUN until apt-get update \ python3-pip rsync procps dh-python quilt bash-completion ; \ do echo "apt failed… trying again in 10s… " ; sleep 10 ; done -RUN echo 'deb http://deb.debian.org/debian sid main' >> /etc/apt/sources.list - RUN until apt-get update \ && apt-get install -y --no-install-recommends openjdk-11-jdk openjdk-17-jdk ; \ do echo "apt failed… trying again in 10s… " ; sleep 10 ; done diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh index 783f914c72..34b17fc4e9 100755 --- a/.build/docker/run-tests.sh +++ b/.build/docker/run-tests.sh @@ -208,6 +208,7 @@ fi # the docker container's env docker_envs="--env TEST_SCRIPT=${test_script} --env JAVA_VERSION=${java_version} --env PYTHON_VERSION=${python_version} --env cython=${cython} --env ANT_OPTS=\"${ANT_OPTS}\"" +[ $DEBUG ] && docker_envs="${docker_envs} --env DEBUG=1" split_str="0_0" if [[ "${split_chunk}" =~ ^[0-9]+/[0-9]+$ ]]; then diff --git a/.build/run-python-dtests.sh b/.build/run-python-dtests.sh index 03080476c2..8a6995dcfb 100755 --- a/.build/run-python-dtests.sh +++ b/.build/run-python-dtests.sh @@ -25,6 +25,7 @@ # ################################ +[ $DEBUG ] && set -x # help if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ "$1" == "-h" ]; then diff --git a/.build/run-tests.sh b/.build/run-tests.sh index e36cd77d6d..7529af933a 100755 --- a/.build/run-tests.sh +++ b/.build/run-tests.sh @@ -19,6 +19,8 @@ # Wrapper script for running a split or regexp of tests (excluding python dtests) # +[ $DEBUG ] && set -x + set -o errexit set -o pipefail diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index 8f497aff0b..b0315c04f6 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -106,6 +106,9 @@ pipeline { } } post { + failure { + echo "ERROR pipeline failed – not all tests were run" + } always { sendNotifications() } @@ -334,14 +337,16 @@ def build(command, cell) { def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz" def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'" - def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true - dir("build") { - archiveArtifacts artifacts: "${logfile}", fingerprint: true - copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/") - } - if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } - if ("jar" == cell.step) { // TODO only stash the project built files. all dependency libraries are restored from the local maven repo using `ant resolver-dist-lib` - stash name: "${cell.arch}_${cell.jdk}" + timeout(time: 1, unit: 'HOURS') { + def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true + dir("build") { + archiveArtifacts artifacts: "${logfile}", fingerprint: true + copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/") + } + if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } + if ("jar" == cell.step) { + stash name: "${cell.arch}_${cell.jdk}" + } } dir("build") { copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/") @@ -376,13 +381,15 @@ def test(command, cell) { script_vars = "${script_vars} cython=\'${cell.cython}\'" } script_vars = fetchDTestsSource(command, script_vars) - buildJVMDTestJars(cell, script_vars, logfile) - def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true - dir("build") { - archiveArtifacts artifacts: "${logfile}", fingerprint: true - copyToNightlies("${logfile}", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) + timeout(time: 1, unit: 'HOURS') { // best throughput with each cell at ~10 minutes + buildJVMDTestJars(cell, script_vars, logfile) + def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true + dir("build") { + archiveArtifacts artifacts: "${logfile}", fingerprint: true + copyToNightlies("${logfile}", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) + } + if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } } - if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } dir("build") { sh """ mkdir -p test/output/${cell.step}