Honour parameter defaults between builds in Jenkinsfile

Also
– increase splits for jvm-dtests, container size and docker limits for simulator-dtest
– print exit status in .build/docker/run-tests.sh and  .build/docker/_docker_init_tests.sh
– add a summary list of test failures in ci_summary.html
– in ubuntu2004_test.docker make sure /home/cassandra exists and has correct perms (from Marcuse)
– when on ci-cassandra, replace use of copyArtifacts in Jenkinsfile generateTestReports() with manual wget of test files (copyArtifacts is notoriously slow, >4hrs in this case)
– copy ci_summary.html and results_details.tar.xz to nightlies
– capture generateTestReports logs
– stream xz where possible, instead of xz compressing afterwards
– fix ant version in centos7-build.docker
– remove docker login (was meangingless, if credentials exist then docker is logged in)
– prefetch docker images from jfrog (to reduce dockerhub pull rate limits), relates to CASSANDRA-18931
– use scripts from cassandra-builds to clean and report on agents, relates to CASSANDRA-18130
– use groovy elvis operators to avoid NPEs/failures on blank param values
– parallel jfrog pulls and cache it in jfrog too
– pin alpine docker image version
– don't stash .git (was causing AccessDeniedException on some unstash), needed fix so deb/rpm packaging worked on not-git work directories
– durabilityHint needs PERFORMANCE_OPTIMIZED, controller becomes scale bottleneck and breaks without it

 patch by Mick Semb Wever; reviewed by Brandon Williams for CASSANDRA-19558

.build/ci/ci_parser.py

@@ -201,8 +201,6 @@ def process_test_cases(suite: JUnitTestSuite, file_name: str, root) -> int:
     return test_count
 
 
-# TODO: Update this to instead be "create_summary_file" and build the entire summary page, not just append failures to existing
-# This should be trivial to do using JUnitTestSuite.failed, passed, etc methods
 def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, output: str) -> None:
     """
     Will create a table with all failed tests in it organized by sorted suite name.
@@ -218,6 +216,8 @@ def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, outpu
     with open(output, 'r') as file:
         soup = BeautifulSoup(file, 'html.parser')
 
+    failures_list_tag = soup.new_tag("div")
+    failures_list_tag.string = '<br/><br/>[Test Failures]<br/><br/>'
     failures_tag = soup.new_tag("div")
     failures_tag.string = '<br/><br/>[Test Failure Details]<br/><br/>'
     suites_tag = soup.new_tag("div")
@@ -250,6 +250,7 @@ def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, outpu
             failures_builder.label_columns(['Class', 'Method', 'Output', 'Duration'], ["width: 15%; text-align: left;", "width: 15%; text-align: left;", "width: 60%; text-align: left;", "width: 10%; text-align: right;"])
             for test in suite.get_tests(JUnitTestStatus.FAILURE):
                 failures_builder.add_row(test.row_data())
+            failures_list_tag.append(BeautifulSoup(failures_builder.build_list(), 'html.parser'))
             failures_tag.append(BeautifulSoup(failures_builder.build_table(), 'html.parser'))
             total_failure_count += failure_count
             if total_failure_count > 200:
@@ -269,6 +270,7 @@ def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, outpu
         """
 
     soup.body.append(totals_tag)
+    soup.body.append(failures_list_tag)
     soup.body.append(failures_tag)
     suites_tag.append(BeautifulSoup(suites_builder.build_table(), 'html.parser'))
     soup.body.append(suites_tag)

.build/ci/junit_helpers.py

@@ -101,6 +101,14 @@ class JUnitResultBuilder:
         </table>
         ''')
 
+        self._list_template = Template('''
+        <div>
+          {% for row in rows %}
+           &nbsp; &nbsp;  {{ row[0] }} {{ row[1] }}<br/>
+          {% endfor %}
+        </div>
+        ''')
+
     @staticmethod
     def add_style_tags(soup: BeautifulSoup) -> None:
         """
@@ -130,6 +138,9 @@ class JUnitResultBuilder:
             raise AssertionError(f'Got invalid number of columns on add_row: {len(row)}. Expected: 4.')
         self._rows.append(row)
 
+    def build_list(self) -> str:
+        return self._list_template.render(rows=self._rows)
+
     def build_table(self) -> str:
         return self._template.render(header=f'{self._name}', labels=self._labels, column_styles=self._column_styles, rows=self._rows)
 

.build/docker/_build-debian.sh

@@ -96,7 +96,7 @@ else
     # if CASSANDRA_VERSION is -alphaN, -betaN, -rcN, it fails on the '-' char; replace with '~'
     CASSANDRA_VERSION=${buildxml_version/-/\~}
     dt=`date +"%Y%m%d"`
-    ref=`git rev-parse --short HEAD`
+    ref=`git rev-parse --short HEAD || grep -q GitSHA src/resources/org/apache/cassandra/config/version.properties && grep GitSHA src/resources/org/apache/cassandra/config/version.properties | cut -d"=" -f2 || echo unknown`
     CASSANDRA_REVISION="${dt}git${ref}"
     dch -D unstable -v "${CASSANDRA_VERSION}-${CASSANDRA_REVISION}" --package "cassandra" "building ${CASSANDRA_VERSION}-${CASSANDRA_REVISION}"
 fi
@@ -120,11 +120,12 @@ fi
 # build package
 dpkg-buildpackage -rfakeroot -uc -us -tc --source-option=--tar-ignore=.git
 
+set +e
 # Move created artifacts to dist dir mapped to docker host directory (must have proper permissions)
 mv ../cassandra[-_]*${CASSANDRA_VERSION}* "${DIST_DIR}"
 # clean build deps
 rm -f cassandra-build-deps_*
 # restore debian/changelog
-git restore debian/changelog
+git restore debian/changelog || true
 
 popd >/dev/null

.build/docker/_build-redhat.sh

@@ -104,7 +104,7 @@ else
    # from the branch name. In this case, fall back to version specified in build.xml.
    CASSANDRA_VERSION="${buildxml_version}"
    dt=`date +"%Y%m%d"`
-   ref=`git rev-parse --short HEAD`
+   ref=`git rev-parse --short HEAD || grep -q GitSHA src/resources/org/apache/cassandra/config/version.properties && grep GitSHA src/resources/org/apache/cassandra/config/version.properties | cut -d"=" -f2 || echo unknown`
    CASSANDRA_REVISION="${dt}git${ref}"
 fi
 

.build/docker/_docker_init_tests.sh

@@ -44,4 +44,5 @@ fi
 # these happen when the image hasn't pre-downloaded all the ccm versions used in tests
 rm -rf /tmp/ccm-*.tar.gz
 
+set -x
 exit ${status}

.build/docker/_docker_run.sh

@@ -95,15 +95,20 @@ image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}"
 
 # Look for existing docker image, otherwise build
 if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
-  # try docker login to increase dockerhub rate limits
+  echo "Build image not found locally, pulling image ${image_name}..."
-  timeout -k 5 5 docker login >/dev/null
   if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
     # Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
+    echo "Building docker image..."
     until docker build -t ${image_name} -f docker/${dockerfile} .  ; do
         echo "docker build failed… trying again in 10s… "
         sleep 10
     done
+    echo "Docker image ${image_name} has been built"
+  else
+    echo "Successfully pulled build image."
   fi
+else
+  echo "Found build image locally."
 fi
 
 # Run build script through docker

.build/docker/centos7-build.docker

@@ -23,7 +23,7 @@ ENV BUILD_HOME=/home/build
 ENV RPM_BUILD_DIR=$BUILD_HOME/rpmbuild
 ENV DIST_DIR=/dist
 ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
-ENV ANT_VERSION=1.10.12
+ENV ANT_VERSION=1.10.14
 ARG UID_ARG=1000
 ARG GID_ARG=1000
 

.build/docker/run-tests.sh

@@ -30,6 +30,12 @@ if [ "$#" -lt 1 ] || [ "$#" -gt 3 ] || [ "$1" == "-h" ]; then
     exit 1
 fi
 
+error() {
+    echo >&2 $2;
+    set -x
+    exit $1
+}
+
 # variables, with defaults
 [ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname "$0")/../..)"
 [ "x${cassandra_dtest_dir}" != "x" ] || cassandra_dtest_dir="${cassandra_dir}/../cassandra-dtest"
@@ -39,12 +45,12 @@ fi
 [ -d "${m2_dir}" ] || { mkdir -p "${m2_dir}" ; }
 
 # pre-conditions
-command -v docker >/dev/null 2>&1 || { echo >&2 "docker needs to be installed"; exit 1; }
+command -v docker >/dev/null 2>&1 || { error 1 "docker needs to be installed"; }
-command -v bc >/dev/null 2>&1 || { echo >&2 "bc needs to be installed"; exit 1; }
+command -v bc >/dev/null 2>&1 || { error 1 "bc needs to be installed"; }
-command -v timeout >/dev/null 2>&1 || { echo >&2 "timeout needs to be installed"; exit 1; }
+command -v timeout >/dev/null 2>&1 || { error 1 "timeout needs to be installed"; }
-(docker info >/dev/null 2>&1) || { echo >&2 "docker needs to running"; exit 1; }
+(docker info >/dev/null 2>&1) || { error 1 "docker needs to running"; }
-[ -f "${cassandra_dir}/build.xml" ] || { echo >&2 "${cassandra_dir}/build.xml must exist"; exit 1; }
+[ -f "${cassandra_dir}/build.xml" ] || { error 1 "${cassandra_dir}/build.xml must exist"; }
-[ -f "${cassandra_dir}/.build/run-tests.sh" ] || { echo >&2 "${cassandra_dir}/.build/run-tests.sh must exist"; exit 1; }
+[ -f "${cassandra_dir}/.build/run-tests.sh" ] || { error 1 "${cassandra_dir}/.build/run-tests.sh must exist"; }
 
 # arguments
 target=$1
@@ -63,8 +69,7 @@ fi
 
 regx_java_version="(${java_version_supported//,/|})"
 if [[ ! "${java_version}" =~ $regx_java_version ]]; then
-    echo "Error: Java version is not in ${java_version_supported}, it is set to ${java_version}"
+    error 1 "Error: Java version is not in ${java_version_supported}, it is set to ${java_version}"
-    exit 1
 fi
 
 # allow python version override, otherwise default to current python version or 3.8
@@ -88,14 +93,10 @@ docker_mounts="${docker_mounts} -v "${build_dir}/tmp":/home/cassandra/cassandra/
 
 # Look for existing docker image, otherwise build
 if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
-  echo "Build image not found locally"
+  echo "Build image not found locally, pulling image ${image_name}..."
-  # try docker login to increase dockerhub rate limits
-  echo "Attempting 'docker login' to increase dockerhub rate limits"
-  timeout -k 5 5 docker login >/dev/null 2>/dev/null
-  echo "Pulling build image..."
   if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
     # Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
-    echo "Building docker image ${image_name}..."
+    echo "Building docker image..."
     until docker build -t ${image_name} -f docker/${dockerfile} .  ; do
       echo "docker build failed… trying again in 10s… "
       sleep 10
@@ -103,9 +104,9 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
     echo "Docker image ${image_name} has been built"
   else
     echo "Successfully pulled build image."
-  fi  
+  fi
 else
-    echo "Found build image locally."
+  echo "Found build image locally."
 fi
 
 pushd ${cassandra_dir} >/dev/null
@@ -119,47 +120,47 @@ if [[ ! -z ${JENKINS_URL+x} ]] && [[ ! -z ${NODE_NAME+x} ]] ; then
 fi
 
 # find host's available cores and mem
-cores=$(docker run --rm alpine nproc --all) || { echo >&2 "Unable to check available CPU cores"; exit 1; }
+cores=$(docker run --rm alpine:3.19.1 nproc --all) || { error 1 "Unable to check available CPU cores"; }
 
 case $(uname) in
     "Linux")
-        mem=$(docker run --rm alpine free -b | grep Mem: | awk '{print $2}') || { echo >&2 "Unable to check available memory"; exit 1; }
+        mem=$(docker run --rm alpine:3.19.1 free -b | grep Mem: | awk '{print $2}') || { error 1 "Unable to check available memory"; }
         ;;
     "Darwin")
-        mem=$(sysctl -n hw.memsize) || { echo >&2 "Unable to check available memory"; exit 1; }
+        mem=$(sysctl -n hw.memsize) || { error 1 "Unable to check available memory"; }
         ;;
     *)
-        echo >&2 "Unsupported operating system, expected Linux or Darwin"
+        error 1 "Unsupported operating system, expected Linux or Darwin"
-        exit 1
 esac
 
 # figure out resource limits, scripts, and mounts for the test type
+docker_flags="-m 5g --memory-swap 5g"
 case ${target} in
     "build_dtest_jars")
     ;;
     "stress-test" | "fqltool-test" )
-        [[ ${mem} -gt $((1 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 1g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
+        [[ ${mem} -gt $((1 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 1g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
     ;;
     # test-burn doesn't have enough tests in it to split beyond 8, and burn and long we want a bit more resources anyway
     "microbench" | "test-burn" | "long-test" | "cqlsh-test" )
-        [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
+        [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
     ;;
-    "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" )
+    "simulator-dtest")
-        [ -f "${cassandra_dtest_dir}/dtest.py" ] || { echo >&2 "${cassandra_dtest_dir}/dtest.py must exist"; exit 1; }
+        [[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
-        [[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
+        docker_flags="-m 15g --memory-swap 15g"
+    ;;
+    "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large")
+        [ -f "${cassandra_dtest_dir}/dtest.py" ] || { error 1 "${cassandra_dtest_dir}/dtest.py not found. please specify 'cassandra_dtest_dir' to point to the local cassandra-dtest source"; }
         test_script="run-python-dtests.sh"
         docker_mounts="${docker_mounts} -v ${cassandra_dtest_dir}:/home/cassandra/cassandra-dtest"
-        # check that ${cassandra_dtest_dir} is valid
+        [[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
-        [ -f "${cassandra_dtest_dir}/dtest.py" ] || { echo >&2 "${cassandra_dtest_dir}/dtest.py not found. please specify 'cassandra_dtest_dir' to point to the local cassandra-dtest source"; exit 1; }
+        docker_flags="-m 15g --memory-swap 15g"
     ;;
-    "test"| "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode" | "simulator-dtest")
+    "test"| "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
-        [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
+        [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
-        max_docker_runs_by_cores=$( echo "sqrt( ${cores} / ${jenkins_executors} )" | bc )
-        max_docker_runs_by_mem=$(( ${mem} / ( 5 * 1024 * 1024 * 1024 * ${jenkins_executors} ) ))
     ;;
     *)
-    echo "unrecognized test type \"${target}\""
+    error 1 "unrecognized test type \"${target}\""
-    exit 1
     ;;
 esac
 
@@ -171,12 +172,8 @@ if (( $(echo "${docker_cpus} > ${docker_cpus_limit}" |bc -l) )) ; then
 fi
 
 # hack: long-test does not handle limited CPUs
-if [ "${target}" == "long-test" ] ; then
+if [ "${target}" != "long-test" ] ; then
-    docker_flags="-m 5g --memory-swap 5g"
+    docker_flags="--cpus=${docker_cpus} ${docker_flags}"
-elif [[ "${target}" =~ dtest* ]] ; then
-    docker_flags="--cpus=${docker_cpus} -m 15g --memory-swap 15g"
-else
-    docker_flags="--cpus=${docker_cpus} -m 5g --memory-swap 5g"
 fi
 
 docker_flags="${docker_flags} -d --rm"
@@ -203,7 +200,7 @@ esac
 
 # cython can be used for cqlsh-test
 if [ "$cython" == "yes" ]; then
-    [ "${target}" == "cqlsh-test" ] || { echo "cython is only supported for cqlsh-test"; exit 1; }
+    [ "${target}" == "cqlsh-test" ] || { error 1 "cython is only supported for cqlsh-test"; }
     ANT_OPTS="${ANT_OPTS}_cython"
 else
     cython="no"
@@ -268,4 +265,5 @@ xz -f ${logfile} 2>/dev/null
 
 popd >/dev/null
 popd >/dev/null
+set -x
 exit ${status}

.build/docker/ubuntu2004_test.docker

@@ -85,6 +85,7 @@ RUN echo "cassandra-tmp ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/build
 RUN chmod 0440 /etc/sudoers.d/build
 
 # switch to the cassandra user
+RUN mkdir -p ${BUILD_HOME} && chmod a+rwx ${BUILD_HOME}
 USER cassandra-tmp
 ENV HOME ${BUILD_HOME}
 WORKDIR ${BUILD_HOME}

.build/run-tests.sh

@@ -172,7 +172,7 @@ _main() {
   fi
 
   # check project is already built. no cleaning is done, so jenkins unstash works, beware.
-  [[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR})"; exit 1; }
+  [[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR} | xargs)"; exit 1; }
 
   # check if dist artifacts exist, this breaks the dtests
   [[ -d "${DIST_DIR}/dist" ]] && { echo "tests don't work when build/dist ("${DIST_DIR}/dist") exists (from \`ant artifacts\`)"; exit 1; }

.jenkins/Jenkinsfile

@@ -48,6 +48,8 @@
 //  will be ignored when run on other environments.
 // Note there are also differences when CI is being run pre- or post-commit.
 //
+// CAUTION! When running CI with changes in this file, ensure the "Pipeline script from SCM" scm details match
+//  the brances being tested. These details don't honour the per-build repository and branch parameterisation.
 //
 // Validate/lint this file using the following command
 // `curl -X POST  -F "jenkinsfile=<.jenkins/Jenkinsfile" https://ci-cassandra.apache.org/pipeline-model-converter/validate`
@@ -56,18 +58,22 @@
 
 pipeline {
   agent { label 'cassandra-small' }
+  options {
+    // must have: avoids agents waste in idle time on controller bottleneck
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+  }
   parameters {
-    string(name: 'repository', defaultValue: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository')
+    string(name: 'repository', defaultValue: params.repository ?: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository')
-    string(name: 'branch', defaultValue: env.BRANCH_NAME, description: 'Branch')
+    string(name: 'branch', defaultValue: params.branch ?: scm.userRemoteConfigs[0].refspec, description: 'Branch')
 
-    choice(name: 'profile', choices: pipelineProfiles().keySet() as List, description: 'Pick a pipeline profile.')
+    choice(name: 'profile', choices: pipelineProfileNames(params.profile ?: ''), description: 'Pick a pipeline profile.')
-    string(name: 'profile_custom_regexp', defaultValue: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*')
+    string(name: 'profile_custom_regexp', defaultValue: params.profile_custom_regexp ?: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*')
 
     choice(name: 'architecture', choices: archsSupported() + "all", description: 'Pick architecture. The ARM64 is disabled by default at the moment.')
-    string(name: 'jdk', defaultValue: "", description: 'Restrict JDK versions. (e.g. "11", "17", etc)')
+    string(name: 'jdk', defaultValue: params.jdk ?: '', description: 'Restrict JDK versions. (e.g. "11", "17", etc)')
 
-    string(name: 'dtest_repository', defaultValue: 'https://github.com/apache/cassandra-dtest' ,description: 'Cassandra DTest Repository')
+    string(name: 'dtest_repository', defaultValue: params.dtest_repository ?: 'https://github.com/apache/cassandra-dtest', description: 'Cassandra DTest Repository')
-    string(name: 'dtest_branch', defaultValue: 'trunk', description: 'DTest Branch')
+    string(name: 'dtest_branch', defaultValue: params.dtest_branch ?: 'trunk', description: 'DTest Branch')
   }
   stages {
     stage('jar') {
@@ -125,6 +131,13 @@ def pipelineProfiles() {
   ]
 }
 
+def pipelineProfileNames(putFirst) {
+    set = pipelineProfiles().keySet() as List
+    set = set - putFirst
+    set.add(0, putFirst)
+    return set
+}
+
 def tasks() {
   // Steps config
   def buildSteps = [
@@ -152,9 +165,9 @@ def tasks() {
     'long-test': [splits: 8],
     'test-oa': [splits: 8],
     'test-system-keyspace-directory': [splits: 8],
-    'jvm-dtest': [splits: 8],
+    'jvm-dtest': [splits: 12],
     'jvm-dtest-upgrade': [splits: 8],
-    'simulator-dtest': [splits: 1],
+    'simulator-dtest': [splits: 1, size: 'large'],
     'dtest': [splits: 64, size: 'large'],
     'dtest-novnode': [splits: 64, size: 'large'],
     'dtest-latest': [splits: 64, size: 'large'],
@@ -221,7 +234,7 @@ def tasks() {
 
 @NonCPS
 def List getMatrixAxes(Map matrix_axes) {
-  List axes = []
+  def List axes = []
   matrix_axes.each { axis, values ->
     List axisList = []
     values.each { value ->
@@ -233,10 +246,10 @@ def List getMatrixAxes(Map matrix_axes) {
 }
 
 def getStepName(cell, command) {
-  arch = "amd64" == cell.arch ? "" : " ${cell.arch}"
+  def arch = "amd64" == cell.arch ? "" : " ${cell.arch}"
-  python = "cqlsh-test" != cell.step ? "" : " python${cell.python}"
+  def python = "cqlsh-test" != cell.step ? "" : " python${cell.python}"
-  cython = "no" == cell.cython ? "" : " cython"
+  def cython = "no" == cell.cython ? "" : " cython"
-  split = command.splits > 1 ? " ${cell.split}/${command.splits}" : ""
+  def split = command.splits > 1 ? " ${cell.split}/${command.splits}" : ""
   return "${cell.step}${arch} jdk${cell.jdk}${python}${cython}${split}"
 }
 
@@ -286,7 +299,7 @@ def isCanonical() {
 }
 
 def isStageEnabled(stage) {
-  return "jar" == stage || pipelineProfiles()[params.profile].contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp)
+  return "jar" == stage || pipelineProfiles()[params.profile]?.contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp)
 }
 
 def isArchEnabled(arch) {
@@ -311,26 +324,24 @@ def build(command, cell) {
       nodeExclusion = "&&!${NODE_NAME}"
       withEnv(cell.collect { k, v -> "${k}=${v}" }) {
         ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
-          cleanAgent(cell.step)
-          cleanWs()
           fetchSource(cell.step, cell.arch, cell.jdk)
           sh """
               test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
               grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
-              """
+             """
+          fetchDockerImages(['almalinux-build', 'bullseye-build', 'centos7-build'])
           def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
-          def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log"
+          def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
           def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
           script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
-          status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee build/${logfile}", returnStatus: true
+          def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
           dir("build") {
-            sh "xz -f *${logfile}"
+            archiveArtifacts artifacts: "${logfile}", fingerprint: true
-            archiveArtifacts artifacts: "${logfile}.xz", fingerprint: true
+            copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
-            copyToNightlies("${logfile}.xz", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
           }
           if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
           if ("jar" == cell.step) { // TODO only stash the project built files. all dependency libraries are restored from the local maven repo using `ant resolver-dist-lib`
-            stash name: "${cell.arch}_${cell.jdk}", useDefaultExcludes: false //, includes: '**/*.jar' //, includes: "*.jar,classes/**,test/classes/**,tools/**"
+            stash name: "${cell.arch}_${cell.jdk}"
           }
           dir("build") {
             copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
@@ -353,11 +364,10 @@ def test(command, cell) {
       nodeExclusion = "&&!${NODE_NAME}"
       withEnv(cell.collect { k, v -> "${k}=${v}" }) {
         ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
-          cleanAgent(cell.step)
-          cleanWs()
           fetchSource(cell.step, cell.arch, cell.jdk)
+          fetchDockerImages(['ubuntu2004_test'])
           def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
-          def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log"
+          def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
           // pipe to tee needs pipefail
           def script_vars = "#!/bin/bash \n set -o pipefail ; "
           script_vars = "${script_vars} python_version=\'${cell.python}\'"
@@ -367,11 +377,10 @@ def test(command, cell) {
           }
           script_vars = fetchDTestsSource(command, script_vars)
           buildJVMDTestJars(cell, script_vars, logfile)
-          status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee -a build/${logfile}", returnStatus: true
+          def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
           dir("build") {
-            sh "xz -f ${logfile}"
+            archiveArtifacts artifacts: "${logfile}", fingerprint: true
-            archiveArtifacts artifacts: "${logfile}.xz", fingerprint: true
+            copyToNightlies("${logfile}", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
-            copyToNightlies("${logfile}.xz", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
           }
           if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
           dir("build") {
@@ -394,17 +403,18 @@ def test(command, cell) {
 }
 
 def fetchSource(stage, arch, jdk) {
-    if ("jar" == stage) {
+  cleanAgent(stage)
-      checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
+  if ("jar" == stage) {
-      sh "mkdir -p build/stage-logs"
+    checkout changelog: false, scm: scmGit(branches: [[name: params.branch ?: 'trunk']], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
-    } else {
+    sh "mkdir -p build/stage-logs"
-      unstash name: "${arch}_${jdk}"
+  } else {
-    }
+    unstash name: "${arch}_${jdk}"
+  }
 }
 
 def fetchDTestsSource(command, script_vars) {
   if (command.containsKey('python-dtest')) {
-    checkout changelog: false, poll: false, scm: scmGit(branches: [[name: params.dtest_branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true), [$class: 'RelativeTargetDirectory', relativeTargetDir: "${WORKSPACE}/build/cassandra-dtest"]], userRemoteConfigs: [[url: params.dtest_repository]])
+    checkout changelog: false, poll: false, scm: scmGit(branches: [[name: params.dtest_branch ?: 'trunk']], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true), [$class: 'RelativeTargetDirectory', relativeTargetDir: "${WORKSPACE}/build/cassandra-dtest"]], userRemoteConfigs: [[url: params.dtest_repository]])
     sh "test -f build/cassandra-dtest/requirements.txt || { echo 'Invalid cassandra-dtest fork/branch'; exit 1; }"
     return "${script_vars} cassandra_dtest_dir='${WORKSPACE}/build/cassandra-dtest'"
   }
@@ -416,12 +426,29 @@ def buildJVMDTestJars(cell, script_vars, logfile) {
     try {
       unstash name: "jvm_dtests_${cell.arch}_${cell.jdk}"
     } catch (error) {
-      sh label: "RUNNING build_dtest_jars...", script: "${script_vars} .build/docker/run-tests.sh build_dtest_jars ${cell.jdk} 2>&1 | tee build/${logfile}"
+      sh label: "RUNNING build_dtest_jars...", script: "${script_vars} .build/docker/run-tests.sh build_dtest_jars ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )"
       stash name: "jvm_dtests_${cell.arch}_${cell.jdk}", includes: '**/dtest*.jar'
     }
   }
 }
 
+def fetchDockerImages(dockerfiles) {
+  // prefetch, from apache jfrog, reduces risking dockerhub pull rate limits
+  // also prefetch alpine:latest as its used as a utility in the scripts
+  def dockerfilesVar = dockerfiles.join(' ')
+  sh """#!/bin/bash
+      for dockerfile in ${dockerfilesVar} ; do
+        image_tag="\$(md5sum .build/docker/\${dockerfile}.docker | cut -d' ' -f1)"
+        image_name="apache/cassandra-\${dockerfile}:\${image_tag}"
+        if ! ( [[ "" != "\$(docker images -q \${image_name} 2>/dev/null)" ]] ) ; then
+          docker pull -q apache.jfrog.io/cassan-docker/\${image_name} &
+        fi
+      done
+      docker pull -q apache.jfrog.io/cassan-docker/alpine:3.19.1 &
+      wait
+     """
+}
+
 def getNodeLabel(command, cell) {
   echo "using node label: cassandra-${cell.arch}-${command.size}"
   return "cassandra-${cell.arch}-${command.size}"
@@ -434,13 +461,13 @@ def copyToNightlies(sourceFiles, remoteDirectory='') {
     retry(9) {
       if (attempt > 1) { sleep(60 * attempt) }
       sshPublisher(
-      continueOnError: true, failOnError: false,
+        continueOnError: true, failOnError: false,
-      publishers: [
+        publishers: [
-        sshPublisherDesc(
+          sshPublisherDesc(
-        configName: "Nightlies",
+          configName: "Nightlies",
-        transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ]
+          transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ]
-        )
+          )
-      ])
+        ])
     }
     echo "archived to https://nightlies.apache.org/${remotePath}"
   }
@@ -449,13 +476,32 @@ def copyToNightlies(sourceFiles, remoteDirectory='') {
 def cleanAgent(job_name) {
   sh "hostname"
   if (isCanonical()) {
-    def maxJobHours = 12
+    def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/"
-    echo "Cleaning project, and pruning docker for '${job_name}' on ${NODE_NAME}…" ;
+    cleanAgentDocker(job_name, agentScriptsUrl)
-    sh """
+    logAgentInfo(job_name, agentScriptsUrl)
-        git clean -qxdff -e build/test/jmh-result.json || true;
-        if pgrep -xa docker || pgrep -af "build/docker" || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --all --force --filter "until=${maxJobHours}h" || true ; else  docker system prune --force --volumes || true ;  fi;
-      """
   }
+  cleanWs()
+}
+
+def cleanAgentDocker(job_name, agentScriptsUrl) {
+  // we don't expect any build to have been running for longer than maxBuildHours
+  def maxBuildHours = 12
+  echo "Pruning docker for '${job_name}' on ${NODE_NAME}…" ;
+  sh """#!/bin/bash
+      set +e
+      wget -q ${agentScriptsUrl}/docker_image_pruner.py
+      wget -q ${agentScriptsUrl}/docker_agent_cleaner.sh
+      bash docker_agent_cleaner.sh ${maxBuildHours}
+    """
+}
+
+def logAgentInfo(job_name, agentScriptsUrl) {
+  sh """#!/bin/bash
+      set +e -o pipefail
+      wget -q ${agentScriptsUrl}/agent_report.sh
+      bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt
+     """
+  copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/")
 }
 
 /////////////////////////////////////////
@@ -463,18 +509,29 @@ def cleanAgent(job_name) {
 /////////////////////////////////////////
 
 def generateTestReports() {
-  // built-in on ci-cassandra will be much faster (local transfer for copyArtifacts and archiveArtifacts)
+  node("cassandra-medium") {
-  def nodeName = isCanonical() ? "built-in" : "cassandra-medium"
+    cleanAgent("generateTestReports")
-  node(nodeName) {
-    cleanWs()
     checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
-    copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
+    def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_generateTestReports.log.xz"
+    sh "mkdir -p build/stage-logs"
+    def teeSuffix = "2>&1 | tee >( xz -c > build/${logfile} )"
+    def script_vars = "#!/bin/bash -x \n "
+    if (isCanonical()) {
+      // copyArtifacts takes >4hrs, hack with manual download
+      sh """${script_vars}
+          ( mkdir -p build/test
+          wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip
+          unzip -x -d build/test -q output.zip ) ${teeSuffix}
+         """
+    } else {
+      copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
+    }
     if (fileExists('build/test/output')) {
       // merge splits for each target's test report, other axes are kept separate
       //   TODO parallelised for loop
       //   TODO results_details.tar.xz needs to include all logs for failed tests
-      sh """
+      sh """${script_vars} (
-          find build/test/output -name *.xml.xz -exec sh -c 'xz -f --decompress {} &' ';' ; wait
+          find build/test/output -name *.xml.xz -exec sh -c 'xz -f --decompress {} &' ';'  ; wait
 
           for target in \$(ls build/test/output/) ; do
             if test -d build/test/output/\${target} ; then
@@ -488,12 +545,13 @@ def generateTestReports() {
 
           .build/docker/_docker_run.sh bullseye-build.docker ci/generate-ci-summary.sh || echo "failed generate-ci-summary.sh"
 
-          tar -cf build/results_details.tar -C build/test/ reports && xz -9f build/results_details.tar
+          tar -cf build/results_details.tar -C build/test/ reports
+          xz -8f build/results_details.tar ) ${teeSuffix}
           """
 
       dir('build/') {
-        archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz", fingerprint: true
+        archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true
-        copyToNightlies('results_details.tar.xz')
+        copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile}')
       }
     }
   }
@@ -502,6 +560,7 @@ def generateTestReports() {
 def sendNotifications() {
   if (isPostCommit() && isCanonical()) {
     // the following is expected only to work on ci-cassandra.apache.org
+    def changes = '?'
     try {
       script {
         changes = formatChangeLogChanges(currentBuild.changeSets)

build.xml

@@ -1851,7 +1851,7 @@
           </fileset>
           <report todir="${build.test.report.dir}/unitTestReport/html" unless:true="${print-summary.skip}" />
       </junitreport>
-      <!-- concat the report through a filter chain to extract what you want -->
+      <!-- concat the report through a filter chain to extract a single line summary -->
       <concat unless:true="${print-summary.skip}">
           <fileset file="${build.test.report.dir}/unitTestReport/html/overview-summary.html" />
           <filterchain>