Merge branch 'cassandra-5.0' into cassandra-6.0

* cassandra-5.0:
  Preload dependencies in build docker images to reduce downloading at build/test/ci runtime

.build/build-accord.xml

@@ -40,6 +40,7 @@
         <arg value="-Paccord_group=org.apache.cassandra" />
         <arg value="-Paccord_artifactId=cassandra-accord" />
         <arg value="-Paccord_version=${version}" />
+        <arg value="-Dmaven.repo.local=${maven.repo.local}" if:set="maven.repo.local" />
       </exec>
       <!-- when dependencies are copied into the build dir they may be ignored if already present, so cleanup to pick up latest changes -->
       <delete>

.build/build-resolver.xml

@@ -56,8 +56,8 @@
 
         <typedef uri="antlib:org.apache.maven.resolver.ant" resource="org/apache/maven/resolver/ant/antlib.xml" classpathref="resolver-ant-tasks.classpath" />
         <resolver:remoterepos id="all">
-            <remoterepo id="resolver-central" url="${artifact.remoteRepository.central}"/>
+            <remoterepo id="resolver-central" url="${artifact.remoteRepository.central}" checksums="fail" />
-            <remoterepo id="resolver-apache" url="${artifact.remoteRepository.apache}"/>
+            <remoterepo id="resolver-apache" url="${artifact.remoteRepository.apache}" checksums="fail" />
             <!-- Snapshot artifacts must not exist in nor be downloaded by any Cassandra release artifact.
                  Please validate that all artifacts included in parent-pom-template.xml are release
                  artifacts before committing.

.build/docker/_create_user.sh

@@ -51,6 +51,12 @@ echo "${username} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${username}
 chmod 0440 /etc/sudoers.d/${username}
 mkdir -p ${BUILD_HOME}/docker ${DIST_DIR} ${BUILD_HOME}/.ssh
 
+# rsync in cached maven dependencies
+echo "Syncing maven dependencies and gradle wrapper"
+rsync -a /home/image-cache/.m2/repository/ ${BUILD_HOME}/.m2/repository/
+cp -a /home/image-cache/.gradle ${BUILD_HOME}/
+chown -R ${username}:${username} ${BUILD_HOME}/.gradle ${BUILD_HOME}/.m2
+
 # we need to make SSH less strict to prevent various dtests from failing when they attempt to
 # git clone a given commit/tag/etc
 echo 'Host *\n UserKnownHostsFile /dev/null\n StrictHostKeyChecking no' > ${BUILD_HOME}/.ssh/config

.build/docker/_docker_run.sh

@@ -31,6 +31,8 @@
 # variables, with defaults
 [ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname -- "$0")/../..)"
 [ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build"
+# parameterise the maven repository host directory, as it cannot be shared across containers
+# m2_dir fails under /tmp on macos
 [ "x${m2_dir}" != "x" ] || m2_dir="${HOME}/.m2/repository"
 [ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; }
 [ -d "${m2_dir}" ] || { mkdir -p "${m2_dir}" ; }
@@ -99,7 +101,7 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
   if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
     # Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
     echo "Building docker image..."
-    until docker build -t ${image_name} -f docker/${dockerfile} .  ; do
+    until docker build -t ${image_name} -f docker/${dockerfile} --load .  ; do
         echo "docker build failed… trying again in 10s… "
         sleep 10
     done

.build/docker/_prepopulate_maven_deps.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Script to prepopulate Maven repository with dependencies from multiple Cassandra branches
+# This will download all dependencies to a custom Maven repository directory
+
+# pre-conditions
+command -v ant >/dev/null 2>&1 || { error 1 "ant needs to be installed"; }
+command -v git >/dev/null 2>&1 || { error 1 "git needs to be installed"; }
+
+
+error() {
+  echo >&2 $2;
+  set -x
+  exit $1
+}
+
+# Function to download dependencies for a branch
+download_deps_for_branch() {
+    local branch=$1
+    local branch_name=$(echo "$branch" | sed 's|origin/||')
+    
+    # Check if branch exists
+    if ! git rev-parse --verify "$branch" >/dev/null 2>&1; then
+        echo "WARNING: Branch $branch does not exist, skipping..."
+        return
+    fi
+    
+    git checkout "$branch"
+    
+    echo ""
+    echo "Downloading dependencies for $branch to $CUSTOM_M2_REPO..."
+    echo ""
+
+    # ensure git modules are initialised
+    ant init
+    # download all dependencies
+    ant -Dmaven.repo.local="$CUSTOM_M2_REPO" -Dlocal.repository="$CUSTOM_M2_REPO" resolver-dist-lib
+}
+
+CUSTOM_M2_REPO="${1:-$HOME/.m2/repository}"
+TMP_DIR=${TMP_DIR:-/tmp}
+
+cd $TMP_DIR
+git clone https://github.com/apache/cassandra.git
+cd cassandra
+git config advice.detachedHead false
+
+# Automatically detect branches from cassandra-5.0 onwards to trunk
+echo "Detecting branches..."
+BRANCHES=()
+
+# Get all origin branches matching cassandra-5.x+, cassandra-6.x+, etc., and trunk
+# Pattern matches: cassandra-5.0, cassandra-5.0.0, cassandra-10.0, cassandra-10.0.1, trunk
+while IFS= read -r branch; do
+    BRANCHES+=("$branch")
+done < <(git branch -r | grep -E "^\s*origin/(cassandra-[5-9][0-9]*\.[0-9]+(\.[0-9]+)?|trunk)$" | sed 's/^[[:space:]]*//' | sort -V)
+
+# If no branches found, fail
+if [ ${#BRANCHES[@]} -eq 0 ]; then
+    echo "ERROR: No branches auto-detected matching pattern origin/cassandra-[5+].x or origin/trunk"
+    echo "Please ensure you have fetched remote branches: git fetch origin"
+    exit 1
+fi
+
+echo "Branches to process:"
+for branch in "${BRANCHES[@]}"; do
+    echo "  - $branch"
+done
+echo "=========================================="
+echo ""
+
+# Create custom Maven repository directory
+mkdir -p "$CUSTOM_M2_REPO"
+
+# Process each branch
+for branch in "${BRANCHES[@]}"; do
+    download_deps_for_branch "$branch"
+done
+
+cd -
+rm -rf $TMP_DIR/cassandra

.build/docker/almalinux-build.docker

@@ -26,7 +26,7 @@ ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
 ARG UID_ARG=1000
 ARG GID_ARG=1000
 
-LABEL org.cassandra.buildenv=almalinux
+LABEL org.cassandra.buildenv=almalinux_build
 
 RUN echo "Building with arguments:" \
     && echo " - DIST_DIR=${DIST_DIR}" \
@@ -100,4 +100,13 @@ RUN sh -c '\
 
 ENV GOROOT="/usr/local/go"
 ENV GOPATH="$BUILD_HOME/go"
-ENV PATH="$PATH:/usr/local/go/bin"
+ENV PATH="$PATH:/usr/local/go/bin"
+
+# Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
+COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
+RUN alternatives --set java $(alternatives --display java | grep "family java-11-openjdk" | cut -d' ' -f1)
+RUN alternatives --set javac $(alternatives --display javac | grep "family java-11-openjdk" | cut -d' ' -f1)
+RUN mkdir -p /home/image-cache && chmod -R a+rwx /home/image-cache
+RUN JAVA_HOME=$(readlink -f /usr/bin/javac | sed "s:/bin/javac::") \
+    bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository && rm /tmp/_prepopulate_maven_deps.sh
+RUN cp -a /root/.gradle /home/image-cache/.gradle

.build/docker/debian-build.docker

@@ -23,7 +23,7 @@ ENV DIST_DIR=/dist
 ENV BUILD_HOME=/home/build
 ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
 
-LABEL org.cassandra.buildenv=bullseye
+LABEL org.cassandra.buildenv=debian_build
 
 RUN echo "Building with arguments:" \
     && echo " - DIST_DIR=${DIST_DIR}" \
@@ -109,4 +109,10 @@ RUN sed -i 's/UID_MIN			 1000/UID_MIN			 100/' /etc/login.defs
 RUN sed -i 's/UID_MIN			 1000/UID_MIN			 10/' /etc/login.defs
 
 # suppress warnings about mismatching ownership
-RUN git config --global --add safe.directory ${CASSANDRA_DIR}
+RUN git config --global --add safe.directory ${CASSANDRA_DIR}
+
+# Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
+COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
+RUN mkdir -p /home/image-cache && chmod -R a+rwx /home/image-cache
+RUN bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository && rm /tmp/_prepopulate_maven_deps.sh
+RUN cp -a /root/.gradle /home/image-cache/.gradle

.build/docker/run-tests.sh

@@ -25,6 +25,8 @@
 [ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname -- "$0")/../..)"
 [ "x${cassandra_dtest_dir}" != "x" ] || cassandra_dtest_dir="${cassandra_dir}/../cassandra-dtest"
 [ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build"
+# parameterise the maven repository host directory, as it cannot be shared across containers.
+# m2_dir fails under /tmp on macos
 [ "x${m2_dir}" != "x" ] || m2_dir="${HOME}/.m2/repository"
 [ "x${docker_timeout_hours}" != "x" ] || docker_timeout_hours="1"
 [ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; }
@@ -147,7 +149,7 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
   if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
     # Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
     echo "Building docker image..."
-    until docker build -t ${image_name} -f docker/${dockerfile} .  ; do
+    until docker build -t ${image_name} -f docker/${dockerfile} --load .  ; do
       echo "docker build failed… trying again in 10s… "
       sleep 10
     done
@@ -294,7 +296,7 @@ docker_command="source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_versi
 # start the container, timeout after 4 hours
 docker_id=$(docker run --name ${container_name} ${docker_flags} ${docker_envs} ${docker_mounts} ${docker_volume_opt} ${image_name} sleep ${docker_timeout_hours}h)
 
-echo "Running container ${container_name} ${docker_id}"
+echo "Running container ${container_name} ${docker_id} using image ${image_name}"
 
 docker exec --user root ${container_name} bash -c "\${CASSANDRA_DIR}/.build/docker/_create_user.sh cassandra $(id -u) $(id -g)" | tee -a ${logfile}
 docker exec --user root ${container_name} update-alternatives --set python /usr/bin/python${python_version} | tee -a ${logfile}

.build/docker/ubuntu-test.docker

@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 LABEL org.opencontainers.image.authors="Apache Cassandra <dev@cassandra.apache.org>"
 
 # CONTEXT is expected to be cassandra/.build
@@ -23,7 +23,7 @@ ENV LC_CTYPE=en_US.UTF-8
 ENV PYTHONIOENCODING=utf-8
 ENV PYTHONUNBUFFERED=true
 
-LABEL org.cassandra.buildenv=ubuntu_2004
+LABEL org.cassandra.buildenv=ubuntu_test
 
 RUN echo "Building with arguments:" \
     && echo " - DIST_DIR=${DIST_DIR}" \
@@ -42,7 +42,7 @@ RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install -y --no-install-recommends software-properties-common apt-utils
+    apt-get install -y --no-install-recommends software-properties-common apt-utils gnupg
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
     add-apt-repository -y ppa:deadsnakes/ppa && \
@@ -54,7 +54,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         vim lsof sudo libjemalloc2 dumb-init locales rsync \
         openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk openjdk-21-jdk ant ant-optional
 
-
+RUN update-alternatives --remove java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/jre/bin/java
+RUN update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/java 1081
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 2
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 3
 RUN python3.8 -m pip install --upgrade pip
@@ -80,6 +81,7 @@ RUN find /etc -type f -name java.security -exec sed -i 's/TLSv1, TLSv1.1//' {} \
 RUN find /etc -type f -name java.security -exec sed -i 's/3DES_EDE_CBC$/3DES_EDE_CBC, TLSv1, TLSv1.1/' {} \;
 
 # create and change to cassandra-tmp user, use an rare uid to avoid collision later on
+RUN mkdir -p /home/image-cache && chmod -R a+rwx /home/image-cache
 RUN adduser --disabled-login --uid 901743 --lastuid 901743 --gecos cassandra cassandra-tmp
 RUN gpasswd -a cassandra-tmp sudo
 RUN echo "cassandra-tmp ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/build
@@ -88,11 +90,16 @@ RUN chmod 0440 /etc/sudoers.d/build
 # switch to the cassandra user
 RUN mkdir -p ${BUILD_HOME} && chmod a+rwx ${BUILD_HOME}
 USER cassandra-tmp
-ENV HOME ${BUILD_HOME}
+ENV HOME=${BUILD_HOME}
 WORKDIR ${BUILD_HOME}
 
 ENV ANT_HOME=/usr/share/ant
 
+# Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
+COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
+RUN bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository
+RUN cp -a /home/cassandra-tmp/.gradle /home/image-cache/.gradle
+
 # run pip commands and setup virtualenv (note we do this after we switch to cassandra user so we
 # setup the virtualenv for the cassandra user and not the root user by accident) for Python 3.8/3.11
 # Don't build cython extensions when installing cassandra-driver. During test execution the driver
@@ -111,15 +118,22 @@ RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
     && pip3 install -r /opt/requirements.txt \
     && pip3 freeze --user"
 
-RUN virtualenv --python=python3.11 ${BUILD_HOME}/env3.11
+RUN python3.11 -m venv ${BUILD_HOME}/env3.11
 RUN chmod +x ${BUILD_HOME}/env3.11/bin/activate
 
 RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
     && source ${BUILD_HOME}/env3.11/bin/activate \
     && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 \
-    && pip3 install -r /opt/requirements.txt \
+    && pip3 install --upgrade \"pip<25.0\"  \"setuptools==60.8.2\" wheel \
+    && pip3 install --no-build-isolation -r /opt/requirements.txt \
     && pip3 freeze --user"
 
+# 4* requires java8, sudo doesn't work on cross-platform builds
+USER root
+RUN update-alternatives --set java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/java
+RUN update-alternatives --set javac /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/javac
+USER cassandra-tmp
+
 # Initialize the CCM git repo as well as this also can fail to clone
 RUN /bin/bash -c "source ${BUILD_HOME}/env3.8/bin/activate && \
     ccm create -n 1 -v git:cassandra-4.1 test && ccm remove test && \
@@ -132,16 +146,23 @@ RUN bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \
     latest_4_1=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.1\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
     for i in $(seq 1 $latest_4_1); do echo $i ; ccm create --quiet -n 1 -v binary:4.1.$i test && ccm remove test ; done'
 
-# 5+ requires java11
+# 5+ requires java11, sudo doesn't work on cross-platform builds
-RUN sudo update-java-alternatives --set java-1.11.0-openjdk-$(dpkg --print-architecture)
+USER root
+RUN update-alternatives --set java /usr/lib/jvm/java-11-openjdk-$(dpkg --print-architecture)/bin/java
+RUN update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-$(dpkg --print-architecture)/bin/javac
+USER cassandra-tmp
 
 # Initialize ccm versions. branch heads and all versions iterating through to the latest version found on downloads.apache.org/cassandra
 RUN rm -fr ${BUILD_HOME}/.ccm/repository/_git_cache_apache
 RUN /bin/bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \
     ccm create --quiet -n 1 -v git:cassandra-5.0 test && ccm remove test && \
+    ccm create --quiet -n 1 -v git:cassandra-6.0 test && ccm remove test && \
     ccm create --quiet -n 1 -v git:trunk test && ccm remove test && \
     latest_5_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")5\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
     for i in $(seq 1 $latest_5_0); do echo $i ; ccm create --quiet -n 1 -v binary:5.0.$i test && ccm remove test ; done'
+    # TODO uncomment when 6.0.0 is released
+    #latest_6_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")6\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
+    #for i in $(seq 1 $latest_6_0); do echo $i ; ccm create --quiet -n 1 -v binary:6.0.$i test && ccm remove test ; done'
 
 # the .git subdirectories to pip installed cassandra-driver breaks virtualenv-clone, so just remove them
 #  and other directories we don't need in image

.build/run-ci

@@ -205,10 +205,10 @@ def parse_arguments() -> argparse.Namespace:
     """
     args = argument_parser().parse_args()
 
-    assert args.repository.startswith("https://github.com/") and args.repository.endswith("cassandra.git"),\
+    assert args.repository.startswith("https://github.com/") and args.repository.removesuffix(".git").endswith("cassandra"),\
         f"Only github apache/cassandra (forked) repository supported, got: {args.repository}"
-    assert args.dtest_repository.startswith("https://github.com/") and args.dtest_repository.endswith("cassandra-dtest.git"),\
+    assert args.dtest_repository.startswith("https://github.com/") and args.dtest_repository.removesuffix(".git").endswith("cassandra-dtest"),\
-        f"Only github apache/cassandra (forked) repository supported, got: {args.dtest_repository}"
+        f"Only github apache/cassandra-dtest (forked) repository supported, got: {args.dtest_repository}"
     assert not (args.setup and args.only_setup), "Both --setup or --only-setup cannot be specified."
     assert not (args.tear_down and args.only_tear_down), "Both --tear-down or --only-tear-down cannot be specified."
     assert not ("custom" == args.profile and not args.profile_custom_regexp), "Custom profile requires --profile-custom-regexp."
@@ -307,27 +307,36 @@ def get_jenkins(k8s_client: client.CoreV1Api, args, kube_ns: str) -> Tuple[str,
     return ip, server
 
 
+def ensure_job_parameters_visible(server: jenkins.Jenkins, job_name: str):
+    """
+    If necessary, triggers a non-parameter build to make parameterised builds visible.
+    """
+    job_info = server.get_job_info(job_name)
+    if any(param.get("parameterDefinitions") for param in job_info.get("property", [])):
+        return
+
+    print(f"Parameters are not visible for job {job_name}; initiating non-parameter build.")
+    queue_item = server.build_job(job_name)
+    build_number = wait_for_build_number(server, queue_item)
+    time.sleep(6)
+    try:
+        server.stop_build(job_name, build_number)
+    except client.exceptions.ApiException as e:
+        print(f"Failed to stop non-parameter build {job_name} {build_number}: {e}")
+    print(f"Parameters should now be available for job {job_name}.")
+
+
+def ensure_cassandra_job_parameters_visible(server: jenkins.Jenkins):
+    """Ensures parameterised builds are visible for all cassandra* jobs."""
+    for job in server.get_jobs():
+        job_name = job.get("name", "")
+        if job_name.startswith("cassandra"):
+            ensure_job_parameters_visible(server, job_name)
+
+
 def trigger_jenkins_build(server: jenkins.Jenkins, job_name: str, **build_params) -> dict:
     """Triggers a Jenkins build with specified parameters and returns the queue item."""
-
+    ensure_job_parameters_visible(server, job_name)
-    def check_for_parameter_build(server: jenkins.Jenkins, job_name: str):
-        """
-        If necessary, triggers a non-parameter build (which makes the parameterised build visible).
-        """
-        job_info = server.get_job_info(job_name)
-        if not any(param.get("parameterDefinitions") for param in job_info.get("property", [])):
-            print("Parameters are not visible; initiating non-parameter build.")
-            queue_item = server.build_job(job_name)
-            build_number = wait_for_build_number(server, queue_item)
-            time.sleep(6)
-            try:
-                server.stop_build(job_name, build_number)
-            except client.exceptions.ApiException as e:
-                print(f"Failed to stop non-parameter build {job_name} {build_number} for job : {e}")
-            print("Parameters should now be available.")
-
-    # Check and trigger non-parameter build if parameters are not visible
-    check_for_parameter_build(server, job_name)
     print("Triggering Jenkins build… ")
     return server.build_job(job_name, parameters=build_params)
 
@@ -820,6 +829,8 @@ def main():
             install_jenkins(args.kubeconfig, args.kubecontext, DEFAULT_KUBE_NS)
 
     (ip, server) = get_jenkins(k8s_client, args, DEFAULT_KUBE_NS)
+    if args.setup or args.only_setup:
+        ensure_cassandra_job_parameters_visible(server)
     if args.only_setup:
         return
     if args.download_results:

.jenkins/Jenkinsfile

@@ -72,7 +72,7 @@ pipeline {
   }
   parameters {
     string(name: 'repository', defaultValue: params.repository ?: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository')
-    string(name: 'branch', defaultValue: params.branch ?: scm.userRemoteConfigs[0].refspec, description: 'Branch')
+    string(name: 'branch', defaultValue: params.branch ?: scm.branch, description: 'Branch')
 
     choice(name: 'profile', choices: pipelineProfileNames(params.profile ?: ''), description: 'Pick a pipeline profile.')
     string(name: 'profile_custom_regexp', defaultValue: params.profile_custom_regexp ?: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*')
@@ -157,6 +157,13 @@ def pipelineProfileNames(putFirst) {
     return set
 }
 
+@NonCPS
+def extractBuildXmlProperty(String xml, String name) {
+    def m = xml =~ /property\s*name="${name}"\s*value="([^"]*)"/
+    assert m, "${name} not found in build.xml"
+    return m[0][1]
+}
+
 @Field Map cachedTasks = null
 
 def tasks() {
@@ -231,12 +238,8 @@ def tasks() {
 
   // find the default JDK and the supported JDKs defined in the build.xml
   def build_xml = readFile(file: 'build.xml')
-  def javaVersionDefaultMatch = (build_xml =~ /property\s*name="java\.default"\s*value="([^"]*)"/)
+  def javaVersionDefault   = extractBuildXmlProperty(build_xml, 'java.default')
-  assert javaVersionDefaultMatch, "java.default property not found in build.xml"
+  def javaVersionsSupported = extractBuildXmlProperty(build_xml, 'java.supported').split(',') as List
-  def javaVersionDefault = javaVersionDefaultMatch[0][1]
-  def javaVersionsSupportedMatch = (build_xml =~ /property\s*name="java\.supported"\s*value="([^"]*)"/)
-  assert javaVersionsSupportedMatch, "java.supported property not found in build.xml"
-  def javaVersionsSupported = javaVersionsSupportedMatch[0][1].split(',') as List
 
   // define matrix axes
   def Map matrix_axes = [
@@ -360,14 +363,29 @@ def build(command, cell) {
           def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
           script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
           timeout(time: 1, unit: 'HOURS') {
-            def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
+            try {
-            dir("build") {
+              def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
-              archiveArtifacts artifacts: "${logfile}", fingerprint: true
+              dir("build") {
-              copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
+                archiveArtifacts artifacts: "${logfile}", fingerprint: true
-            }
+                copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
-            if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
+              }
-            if ("jar" == cell.step) {
+              if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
-              stash name: "${cell.arch}_${cell.jdk}"
+              if ("jar" == cell.step) {
+                stash name: "${cell.arch}_${cell.jdk}"
+              }
+            } catch (exc) {
+              if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
+                def descriptions = []
+                for (def cause in exc.getCauses()) {
+                  echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
+                  if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
+                    throw exc // user explicitly aborted — do not retry
+                  }
+                  descriptions.add(cause.getShortDescription())
+                }
+                error("Retryable interruption: ${descriptions.join(', ')}")
+              }
+              throw exc
             }
           }
           dir("build") {
@@ -414,10 +432,16 @@ def test(command, cell) {
               }
               if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
             } catch (exc) {
-              if (exc.getClass().getName() == "org.jenkinsci.plugins.workflow.steps.FlowInterruptedException") {
+              if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
-                for (def causeOfInterruption in exc.getCauses()) {
+                def descriptions = []
-                  echo "CauseOfInterruption: ${causeOfInterruption.getClass().getName()} - ${causeOfInterruption.getShortDescription()}"
+                for (def cause in exc.getCauses()) {
+                  echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
+                  if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
+                    throw exc // user explicitly aborted — do not retry
+                  }
+                  descriptions.add(cause.getShortDescription())
                 }
+                error("Retryable interruption: ${descriptions.join(', ')}")
               }
               throw exc
             } finally {

.jenkins/k8s/README.md

@@ -24,7 +24,7 @@ ZONE="us-central1-c"
 gcloud container clusters create ${CLUSTER_NAME} --machine-type e2-standard-8 --disk-type=pd-ssd --num-nodes 1 --node-labels=cassandra.jenkins.controller=true --autoscaling-profile optimize-utilization --zone ${ZONE}
 
 # small resource nodes
-gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-4 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE}
+gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE}
 
 # medium resource nodes
 #  preference (by cost): n2-highcpu-8, c3-highcpu-8, n4-highcpu-8, n1-highcpu-16
@@ -32,6 +32,12 @@ gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --mac
 
 # large resource nodes
 gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE}
+
+# For each sized resource nodes, pick any machine type that fits, those listed above should work and be the most cost-effective, but this can change region to region
+# See https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/Jenkinsfile#L35-L38
+# and agent.podTemplates.*.resourceLimitCpu and agent.podTemplates.*.resourceLimitMemory (adding gke/eks requirements) in https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/k8s/jenkins-deployment.yaml
+# The jenkins resource requirements should fit into the corresponding dind podTemplate limits.
+# Remember to allow a buffer for gke/eks pods deployed on each node.
 ```
 
 

.jenkins/k8s/jenkins-deployment.yaml

@@ -36,6 +36,7 @@ controller:
   customJenkinsLabels:
     - controller
   resources:
+    # increase cpu/memory as agent pool sizes get bigger (pre-ci.c.a.o uses 8 and 20g)
     requests:
       cpu: 4
       memory: 16G
@@ -97,6 +98,23 @@ controller:
                   }
                 }
               }
+          - script: >
+              pipelineJob('cassandra-6.0') {
+                definition {
+                  cpsScm {
+                    scm {
+                      git {
+                        remote {
+                          url('https://github.com/apache/cassandra')
+                        }
+                        branch('cassandra-6.0')
+                        scriptPath('.jenkins/Jenkinsfile')
+                      }
+                    }
+                    lightweight()
+                  }
+                }
+              }
           - script: >
               pipelineJob('cassandra-5.0') {
                 definition {
@@ -216,6 +234,19 @@ agent:
           - emptyDirVolume:
               memory: 'false'
               mountPath: /certs
+        # limit one agent pod per node for simpler operations (like orphan cleanup)
+        yaml: |
+          spec:
+            affinity:
+              podAntiAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  - labelSelector:
+                      matchExpressions:
+                        - key: jenkins/cassius-jenkins-agent
+                          operator: In
+                          values:
+                            - "true"
+                    topologyKey: kubernetes.io/hostname
     agent-dind-medium: |
       - name: agent-dind-medium
         label: agent-dind cassandra-medium cassandra-amd64-medium
@@ -293,6 +324,19 @@ agent:
           - emptyDirVolume:
               memory: 'false'
               mountPath: /certs
+        # limit one agent pod per node for simpler operations (like orphan cleanup)
+        yaml: |
+          spec:
+            affinity:
+              podAntiAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  - labelSelector:
+                      matchExpressions:
+                        - key: jenkins/cassius-jenkins-agent
+                          operator: In
+                          values:
+                            - "true"
+                    topologyKey: kubernetes.io/hostname
     agent-dind-large: |
       - name: agent-dind-large
         label: agent-dind cassandra-large cassandra-amd64-large cassandra-amd64-large-dedicated
@@ -370,5 +414,18 @@ agent:
           - emptyDirVolume:
               memory: 'false'
               mountPath: /certs
+        # limit one agent pod per node for simpler operations (like orphan cleanup)
+        yaml: |
+          spec:
+            affinity:
+              podAntiAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  - labelSelector:
+                      matchExpressions:
+                        - key: jenkins/cassius-jenkins-agent
+                          operator: In
+                          values:
+                            - "true"
+                    topologyKey: kubernetes.io/hostname