Jenkins improvements: fail-fast, disk-usage, helm overrides and safety

* Add ephemeral-storage requests and limits to agents, and log peak disk usage
Agents were evicted at the node level due to lack of disk space. Each pod needs minimum 100GiB, and each template now budgets 80Gi of the ~89Gi that a 100GiB node allocates, with docker's own volume carries a 60Gi sizeLimit.  Exceeding either evicts the pod, so logging has been added in each test split before and after the run. All script blocks in Jenkinsfile are now labeled for improved readability in test runs.

* `.build/run-ci` must not silently drop a site's helm values customisations
Long-lived instances like pre-ci.cassandra.apache.org carry values in addition to the jenkins-deployment.yaml, and any `helm upgrade -f jenkins-deployment.yaml` (or `.build/run-ci`) would drop them.  Allow these values to be kept in a separate file, passed in with `--values-override`. Before upgrading helm values, we now compare the deployed values against what is about to be applied and require confirmation for anything only the site holds.  This catches added keys and dropped list items (but it cannot catch every change).
Also keep the jenkins-home pvc claim on uninstall, so --only-tear-down no longer destroys build history. Add jenkins-test.sh and a gha workflow for test validating changes under .jenkins/

 patch by Mick Semb Wever; reviewed by Dmitry Konstantinov for CASSANDRA-21523

.build/build-rat.xml

@@ -72,7 +72,7 @@
                  <exclude name="test/data/jmxdump/cassandra-*-jmx.yaml"/>
                  <!-- Documentation files -->
                  <exclude name=".github/pull_request_template.md"/>
-                 <exclude name=".github/workflows/code-check.yaml"/>
+                 <exclude name=".github/workflows/**.yaml"/>
                  <exclude NAME="doc/modules/**/*"/>
                  <exclude NAME="src/java/**/*.md"/>
                  <exclude NAME="**/README*"/>

.build/run-ci

@@ -25,7 +25,7 @@ Python dependencies are found in .build/run-ci.d/requirements.txt
 Custom environment variables can be set in .build/.run-ci.env
 
 lint with:
- `pylint --disable=C0301,W0511,C0103,W0702,C0415,C0116,C0115,R0914,W0603,R0915,R0913,R0911 run-ci`
+ `pylint --disable=C0301,W0511,C0103,W0702,C0415,C0116,C0115,R0914,W0603,R0915,R0913,R0917,R0911,W0212,W0621 run-ci`
 
 test with:
  `python .build/run-ci.d/run-ci-test.py`
@@ -38,6 +38,7 @@ import gzip
 import itertools
 import os
 import shutil
+import socket
 import subprocess
 import sys
 import tarfile
@@ -50,9 +51,10 @@ from urllib.request import urlretrieve
 from typing import Optional, Tuple
 
 # External Libraries (`pip install -r .build/run-ci.d/requirements.txt`)
+import requests
+import yaml
 from bs4 import BeautifulSoup
 from kubernetes import client, config, stream
-import requests
 
 try:
     import jenkins
@@ -72,9 +74,9 @@ def base_job_name(args) -> str:
     """
     if not hasattr(base_job_name, "_cached_result"):
         raw_url = args.repository.replace("https://github.com/", "https://raw.githubusercontent.com/").removesuffix(".git") + f"/{args.branch}/build.xml"
-        if 200 != requests.head(raw_url).status_code:
+        if 200 != requests.head(raw_url, timeout=30).status_code:
             raise ValueError(f"GitHub unavailable, or this branch has not been pushed yet: {args.repository} @ {args.branch} (or remote tracking not setup up: `git config --get branch.{args.branch}.remote` and `git config --get branch.{args.branch}.merge`)")
-        response = requests.get(raw_url)
+        response = requests.get(raw_url, timeout=30)
         response.raise_for_status()
         for line in response.text.splitlines():
             if 'property' in line and 'name="base.version"' in line:
@@ -97,7 +99,7 @@ def is_local_git_dirty(args) -> bool:
     # use base_job_name to verify the remote branch exists
     base_job_name(args)
     # check if the working directory is clean
-    clean = subprocess.run(["git", "-C", str(CASSANDRA_DIR), "diff-index", "--quiet", "HEAD", "--"]).returncode
+    clean = subprocess.run(["git", "-C", str(CASSANDRA_DIR), "diff-index", "--quiet", "HEAD", "--"], check=False).returncode
     # check if there are unpushed committed changes
     unpushed_commits = bool(subprocess.run(["git", "-C", str(CASSANDRA_DIR), "log", "@{u}..HEAD", "--name-only"],
                                            capture_output=True, text=True, check=False).stdout.strip())
@@ -190,6 +192,7 @@ def argument_parser() -> argparse.ArgumentParser:
     parser.add_argument("-k", "--dtest-branch", default=DEFAULT_DTEST_REPO_BRANCH, help="DTest repository branch.")
     parser.add_argument("-s", "--setup", action="store_true", help="Set up Jenkins before the build.")
     parser.add_argument("--only-setup", action="store_true", help="Only install Jenkins into the k8s cluster.")
+    parser.add_argument("-f", "--values-override", help="Path to an additional helm values file, applied over .jenkins/k8s/jenkins-deployment.yaml. Required when the target cluster carries site customisations, see .jenkins/k8s/README.md")
     parser.add_argument("--tear-down", action="store_true", help="Tear down Jenkins after the build.")
     parser.add_argument("--only-tear-down", action="store_true", help="Only tear down Jenkins.")
     parser.add_argument("--only-node-cleaner", action="store_true", help="Only run the node cleaner. The node cleaner scans the k8s nodes, eagerly terminating those unused.")
@@ -210,6 +213,8 @@ def parse_arguments() -> argparse.Namespace:
     assert not (args.setup and args.only_setup), "Both --setup or --only-setup cannot be specified."
     assert not (args.tear_down and args.only_tear_down), "Both --tear-down or --only-tear-down cannot be specified."
     assert not ("custom" == args.profile and not args.profile_custom_regexp), "Custom profile requires --profile-custom-regexp."
+    assert not (args.values_override and not (args.setup or args.only_setup)), "--values-override requires --setup or --only-setup."
+    assert not (args.values_override and not Path(args.values_override).is_file()), f"No such values override file: {args.values_override}"
 
     if not args.url and os.environ.get("JENKINS_URL"):
         args.url = os.environ.get("JENKINS_URL")
@@ -251,19 +256,113 @@ def run_kubectl_command(kubeconfig: Optional[str], kubecontext: Optional[str], k
     cmd += command
     return subprocess.run(cmd, capture_output=True, text=True, check=True).stdout.strip()
 
-def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ns: str):
+def run_helm_command(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ns: str, command: list,
-    """Installs Jenkins Operator using Helm in the specified K8s namespace."""
+                     capture_output: bool = True, check: bool = True) -> subprocess.CompletedProcess:
-    print("Adding Helm repository for Jenkins Operator...")
+    """Runs a helm command with the specified kubeconfig, context and namespace."""
-    subprocess.run(["helm", "repo", "add", "jenkins", "https://charts.jenkins.io"], check=True)
-    subprocess.run(["helm", "repo", "update"], check=True)
-
     cmd = ["helm"]
     if kubeconfig:
         cmd += ["--kubeconfig", kubeconfig]
     if kubecontext:
         cmd += ["--kube-context", kubecontext]
-    cmd += ["--namespace", kube_ns, "upgrade", "--install", "-f", DEPLOY_YAML, "cassius", "jenkins/jenkins", "--wait"]
+    cmd += ["--namespace", kube_ns]
-    result = subprocess.run(cmd, capture_output=True, check=True)
+    cmd += command
+    return subprocess.run(cmd, capture_output=capture_output, text=True, check=check)
+
+def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ns: str,
+                    values_override: Optional[str] = None):
+    """Installs Jenkins Operator using Helm in the specified K8s namespace."""
+
+    def confirm_helm_updates():
+        """Prompts before an upgrade drops any values the deployed jenkins currently has."""
+
+        def helm_values() -> dict:
+            """
+            The values a deployed jenkins was last installed with, or an empty dict when there is no release.
+            These are the user-supplied values, whatever files they came from, so they include any customisations
+            made to the site outside of `.jenkins/k8s/jenkins-deployment.yaml`.
+            """
+            result = run_helm_command(kubeconfig, kubecontext, kube_ns,
+                                      ["get", "values", "cassius", "-o", "yaml"], check=False)
+            if result.returncode != 0:
+                debug(f"No existing cassius release found in namespace {kube_ns}: {result.stderr.strip()}")
+                return {}
+            return yaml.safe_load(result.stdout) or {}
+
+        def merge_values(base: dict, override: dict) -> dict:
+            """Merges two helm values files the way helm does: maps key by key, everything else replaced."""
+            merged = dict(base)
+            for key, value in override.items():
+                if isinstance(value, dict) and isinstance(merged.get(key), dict):
+                    merged[key] = merge_values(merged[key], value)
+                else:
+                    merged[key] = value
+            return merged
+
+        def detect_lost_values(live: dict, proposed: dict) -> dict:
+            """
+            Values the deployed jenkins has that an upgrade would drop, as {dotted.key.path: live value}.
+
+            A key held live but absent from what is about to be applied is either a customisation made to this site,
+            or a key that `.jenkins/k8s/jenkins-deployment.yaml` has removed since the site was last deployed.
+            Note also what this cannot see: a key that exists in both but was given a different value locally, such
+            as an edited `agent.podTemplates` entry, is silently overwritten.
+            Read the diff of `helm template` before deploying an unfamiliar site.
+            """
+            def leaf_values(values, path: str = "") -> dict:
+                """Flattens a values map to {dotted.key.path: value}, lists are leaves (helm replaces them)."""
+                if not isinstance(values, dict):
+                    return {path: values}
+                leaves = {}
+                for key, value in values.items():
+                    leaves.update(leaf_values(value, f"{path}.{key}" if path else str(key)))
+                return leaves
+
+            live_leaves, proposed_leaves = leaf_values(live), leaf_values(proposed)
+            lost = {path: value for path, value in live_leaves.items() if path not in proposed_leaves}
+            # lists are replaced wholesale, so also report items dropped from a list that is otherwise still there
+            for path, value in live_leaves.items():
+                if isinstance(value, list) and isinstance(proposed_leaves.get(path), list):
+                    dropped = [item for item in value if item not in proposed_leaves[path]]
+                    if dropped:
+                        lost[f"{path}[]"] = dropped
+            return lost
+
+        with open(DEPLOY_YAML, encoding="utf-8") as deploy_yaml:
+            proposed = yaml.safe_load(deploy_yaml) or {}
+        if values_override:
+            with open(values_override, encoding="utf-8") as override_yaml:
+                proposed = merge_values(proposed, yaml.safe_load(override_yaml) or {})
+
+        lost = detect_lost_values(helm_values(), proposed)
+        if not lost:
+            return
+
+        print(f"\nWARNING: {len(lost)} value(s) the deployed jenkins has are absent from what is about to be applied.")
+        print("Each is either a customisation of this site, or a key removed from jenkins-deployment.yaml since"
+              " the site was last deployed.  Upgrading drops them:\n")
+        for path in sorted(lost):
+            value = str(lost[path]).replace("\n", " ")
+            print(f"  {path}: {value[:100] + '…' if len(value) > 100 else value}")
+        print(f"\nTo keep any of them, add them to a values override file (see .jenkins/k8s/README.md) and pass"
+              f" `--values-override`{' (the file passed does not contain them)' if values_override else ''}.")
+
+        if not sys.stdin.isatty():
+            print("Refusing to drop them when running non-interactively.")
+            sys.exit(1)
+        if input("\nDrop these values and continue? [y/N] ").strip().lower() not in ("y", "yes"):
+            print("Aborted, nothing was deployed.")
+            sys.exit(1)
+
+    confirm_helm_updates()
+
+    print("Adding Helm repository for Jenkins Operator...")
+    subprocess.run(["helm", "repo", "add", "jenkins", "https://charts.jenkins.io"], check=True)
+    subprocess.run(["helm", "repo", "update"], check=True)
+
+    # site customisations are applied last, helm merges each -f over the previous
+    values_files = ["-f", DEPLOY_YAML] + (["-f", values_override] if values_override else [])
+    result = run_helm_command(kubeconfig, kubecontext, kube_ns,
+                              ["upgrade", "--install"] + values_files + ["cassius", "jenkins/jenkins", "--wait"])
 
     run_kubectl_command(kubeconfig, kubecontext, kube_ns,
                         ["exec", DEFAULT_POD_NAME, "--",
@@ -275,6 +374,19 @@ def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_
         sys.exit(1)
 
 
+def wait_for_jenkins_http(ip: str):
+    host, port = (ip.rsplit(":", 1)[0], int(ip.rsplit(":", 1)[1])) if ":" in ip else (ip, 80)
+    spin_while(f"Waiting for Jenkins HTTP at {host}:{port}… ", lambda: _tcp_connect_ok(host, port))
+
+
+def _tcp_connect_ok(host: str, port: int) -> bool:
+    try:
+        with socket.create_connection((host, port), timeout=2):
+            return True
+    except OSError:
+        return False
+
+
 def get_jenkins(k8s_client: client.CoreV1Api, args, kube_ns: str) -> Tuple[str, jenkins.Jenkins]:
     """Authenticates to Jenkins and returns the Jenkins ip and server objects."""
 
@@ -780,13 +892,10 @@ def cleanup_and_maybe_teardown(kubeconfig: Optional[str], kubecontext: Optional[
     IS_RUNNING = False
     if tear_down:
         print("Cleaning up Jenkins and all resources.")
-        cmd = ["helm"]
+        run_helm_command(kubeconfig, kubecontext, kube_ns, ["uninstall", "cassius"], capture_output=False)
-        if kubeconfig:
+        # the pvc is annotated `helm.sh/resource-policy: keep`, see .jenkins/k8s/jenkins-deployment.yaml
-            cmd += ["--kubeconfig", kubeconfig]
+        print(f"Jenkins uninstalled. The jenkins-home volume was kept, delete it with:\n"
-        if kubecontext:
+              f"  kubectl --namespace {kube_ns} delete pvc cassius-jenkins")
-            cmd += ["--kube-context", kubecontext]
-        cmd += ["--namespace", kube_ns, "uninstall", "cassius"]
-        subprocess.run(cmd, check=True)
 
 
 @contextmanager
@@ -824,10 +933,11 @@ def main():
     if args.setup or args.only_setup:
         init_k8s_namespace(k8s_client, DEFAULT_KUBE_NS)
         with helm_installation_lock(Path("/tmp/.cassandra-run-ci.lock")):
-            install_jenkins(args.kubeconfig, args.kubecontext, DEFAULT_KUBE_NS)
+            install_jenkins(args.kubeconfig, args.kubecontext, DEFAULT_KUBE_NS, args.values_override)
 
     (ip, server) = get_jenkins(k8s_client, args, DEFAULT_KUBE_NS)
     if args.setup or args.only_setup:
+        wait_for_jenkins_http(ip)
         ensure_cassandra_job_parameters_visible(server)
     if args.only_setup:
         return

.build/run-ci.d/README.md

@@ -3,7 +3,7 @@
 ```
 ➤ .build/run-ci --help
 usage: run-ci [-h] [-c KUBECONFIG] [-x KUBECONTEXT] [-i URL] [-u USER] [-r REPOSITORY] [-b BRANCH] [-p {packaging,skinny,pre-commit,pre-commit w/ upgrades,post-commit,custom}] [-e PROFILE_CUSTOM_REGEXP] [-j JDK] [-d DTEST_REPOSITORY] [-k DTEST_BRANCH]
-              [-s] [--only-setup] [--tear-down] [--only-tear-down] [--only-node-cleaner] [-o DOWNLOAD_RESULTS]
+              [-s] [--only-setup] [-v VALUES_OVERRIDE] [--tear-down] [--only-tear-down] [--only-node-cleaner] [-o DOWNLOAD_RESULTS]
 
 Run CI pipeline for Cassandra on K8s using Jenkins.
 
@@ -30,6 +30,8 @@ options:
                         DTest repository branch.
   -s, --setup           Set up Jenkins before the build.
   --only-setup          Only install Jenkins into the k8s cluster.
+  -v VALUES_OVERRIDE, --values-override VALUES_OVERRIDE
+                        Path to an additional helm values file, applied over .jenkins/k8s/jenkins-deployment.yaml. Required when the target cluster carries site customisations, see .jenkins/k8s/README.md
   --tear-down           Tear down Jenkins after the build.
   --only-tear-down      Only tear down Jenkins.
   --only-node-cleaner   Only run the node cleaner. The node cleaner scans the k8s nodes, eagerly terminating those unused.
@@ -63,7 +65,15 @@ Setup/Update Jenkins Helm into your current kubeconfig
 .build/run-ci --only-setup
 ```
 
-Uninstall Jenkins from your current kubeconfig
+Setup/Update Jenkins Helm into a cluster that carries site customisations, e.g. pre-ci.cassandra.apache.org
+```
+.build/run-ci --only-setup --values-override ~/.cassandra-ci/pre-ci-overrides.yaml
+```
+
+Before any setup, the values already deployed are compared against those about to be applied.  Any value the deployed jenkins holds that the new files lack is listed, and confirmation is asked for before it is dropped; running non-interactively aborts instead.  See `.jenkins/k8s/README.md` for what this can and cannot catch.
+
+Uninstall Jenkins from your current kubeconfig.
 ```
 .build/run-ci --only-tear-down
 ```
+The jenkins-home volume is kept; delete it separately with `kubectl delete pvc cassius-jenkins`

.build/run-ci.d/requirements.txt

@@ -18,6 +18,7 @@ bs4
 dotenv
 kubernetes
 python-jenkins
+pyyaml
 requests
 
 # optional for different clouds

.build/run-ci.d/run-ci-test.py

@@ -27,12 +27,14 @@
 
 import argparse
 from pathlib import Path
+import tempfile
 import unittest
 from unittest.mock import patch, MagicMock
 
 
 # Import the functions from the script
 from run_ci import (
+    DEPLOY_YAML,
     debug,
     install_jenkins,
     get_jenkins,
@@ -55,13 +57,84 @@ class TestCIPipeline(unittest.TestCase):
         debug("Test message")
         mock_print.assert_called_with("Test message")
 
+    # the pre-flight check is nested inside install_jenkins, so it is exercised through it: the mocked
+    # `helm get values` stdout stands in for the values the site already has deployed
+    LIVE_STORAGE_CLASS = "persistence:\n  storageClass: gp2\n"
+
     @patch('run_ci.subprocess.run')
     def test_install_jenkins(self, mock_run):
-        mock_run.return_value = MagicMock(returncode=0)
+        # empty stdout, i.e. nothing deployed yet, so the pre-flight check has nothing to warn about
+        mock_run.return_value = MagicMock(returncode=0, stdout="")
         install_jenkins("test-namespace", Path("/fake/cassandra/dir"), "default")
         mock_run.assert_any_call(["helm", "repo", "add", "jenkins", "https://charts.jenkins.io"], check=True)
         mock_run.assert_any_call(["helm", "repo", "update"], check=True)
 
+    @patch('run_ci.subprocess.run')
+    def test_install_jenkins_values_override(self, mock_run):
+        mock_run.return_value = MagicMock(returncode=0, stdout="")
+        with tempfile.NamedTemporaryFile("w", suffix=".yaml") as override:
+            override.write(self.LIVE_STORAGE_CLASS)
+            override.flush()
+            install_jenkins(None, None, "default", override.name)
+            upgrade_cmd = [c for c in [call.args[0] for call in mock_run.call_args_list] if "upgrade" in c][0]
+            # the site's overrides must come after, and never replace, the repo's deployment yaml
+            self.assertEqual(["-f", DEPLOY_YAML, "-f", override.name], upgrade_cmd[5:9])
+
+    @patch('run_ci.sys.stdin.isatty')
+    @patch('run_ci.print')
+    @patch('run_ci.subprocess.run')
+    def test_install_jenkins_aborts_non_interactively(self, mock_run, mock_print, mock_isatty):
+        mock_isatty.return_value = False
+        mock_run.return_value = MagicMock(returncode=0, stdout=self.LIVE_STORAGE_CLASS)
+        with self.assertRaises(SystemExit):
+            install_jenkins(None, None, "default")
+        self.assertEqual([], [c for c in [call.args[0] for call in mock_run.call_args_list] if "upgrade" in c])
+        # and continues when the customisation is passed back in as an override.  This also proves the merge is
+        # per key: were the override to replace the whole persistence map, its other keys would now be reported lost
+        with tempfile.NamedTemporaryFile("w", suffix=".yaml") as override:
+            override.write(self.LIVE_STORAGE_CLASS)
+            override.flush()
+            install_jenkins(None, None, "default", override.name)
+        self.assertEqual(1, len([c for c in [call.args[0] for call in mock_run.call_args_list] if "upgrade" in c]))
+
+    @patch('run_ci.input')
+    @patch('run_ci.sys.stdin.isatty')
+    @patch('run_ci.print')
+    @patch('run_ci.subprocess.run')
+    def test_install_jenkins_prompts(self, mock_run, mock_print, mock_isatty, mock_input):
+        mock_isatty.return_value = True
+        mock_run.return_value = MagicMock(returncode=0, stdout=self.LIVE_STORAGE_CLASS)
+        mock_input.return_value = "n"
+        with self.assertRaises(SystemExit):
+            install_jenkins(None, None, "default")
+        mock_input.return_value = "y"
+        install_jenkins(None, None, "default")
+
+    @patch('run_ci.sys.stdin.isatty')
+    @patch('run_ci.print')
+    @patch('run_ci.subprocess.run')
+    def test_install_jenkins_reports_only_detectable_losses(self, mock_run, mock_print, mock_isatty):
+        mock_isatty.return_value = False
+        # a plugin only this site installs is reported, as is a key the site alone holds; a key held in both but
+        # locally edited (persistence.size) cannot be seen, and must not be claimed
+        mock_run.return_value = MagicMock(returncode=0, stdout="persistence:\n  size: 1Ti\n"
+                                          "controller:\n  installPlugins:\n    - site-only-plugin\n")
+        with self.assertRaises(SystemExit):
+            install_jenkins(None, None, "default")
+        printed = " ".join(str(call.args[0]) for call in mock_print.call_args_list if call.args)
+        self.assertIn("controller.installPlugins[]", printed)
+        self.assertIn("site-only-plugin", printed)
+        self.assertNotIn("persistence.size", printed)
+
+    @patch('run_ci.print')
+    @patch('run_ci.subprocess.run')
+    def test_install_jenkins_when_nothing_deployed(self, mock_run, mock_print):
+        # `helm get values` fails when there is no release, and nothing is then warned about
+        mock_run.side_effect = lambda cmd, **kwargs: MagicMock(returncode=1 if "get" in cmd else 0,
+                                                              stdout="", stderr="release: not found")
+        install_jenkins(None, None, "default")
+        self.assertEqual([], [call.args[0] for call in mock_print.call_args_list if "WARNING" in str(call.args)])
+
     @patch('run_ci.subprocess.run')
     @patch('run_ci.jenkins.Jenkins')
     def test_get_jenkins(self, mock_jenkins, mock_run):
@@ -91,13 +164,14 @@ class TestCIPipeline(unittest.TestCase):
     @patch('run_ci.stream.stream')
     def test_delete_remote_junit_files(self, mock_stream):
         mock_k8s_client = MagicMock()
-        delete_remote_junit_files(mock_k8s_client, "test-pod", "test-namespace", 456)
+        delete_remote_junit_files(mock_k8s_client, "test-pod", "test-namespace", "test-job", 456)
         mock_stream.assert_called()
 
     @patch('run_ci.subprocess.run')
     def test_cleanup_and_maybe_teardown(self, mock_run):
         cleanup_and_maybe_teardown(None, None, "test-namespace", True)
-        mock_run.assert_called_with(["helm", "--namespace", "test-namespace", "uninstall", "cassius"], check=True)
+        mock_run.assert_called_with(["helm", "--namespace", "test-namespace", "uninstall", "cassius"],
+                                    capture_output=False, text=True, check=True)
 
     @patch('run_ci.fcntl.flock')
     def test_helm_installation_lock(self, mock_flock):

.github/workflows/jenkins-check.yaml

@@ -0,0 +1,58 @@
+name: CI Check
+
+# Checks the CI tooling itself: the run-ci script and everything under .jenkins/
+# Nothing here touches a k8s cluster; the helm and groovy steps only render and parse.
+
+on:
+    push:
+        branches:
+            - '**'
+        paths:
+            - '.build/run-ci'
+            - '.build/run-ci.d/**'
+            - '.jenkins/**'
+            - '.github/workflows/jenkins-check.yaml'
+    pull_request:
+        paths:
+            - '.build/run-ci'
+            - '.build/run-ci.d/**'
+            - '.jenkins/**'
+            - '.github/workflows/jenkins-check.yaml'
+
+jobs:
+    run-ci:
+        runs-on: ubuntu-latest
+
+        steps:
+          - name: Checkout code
+            uses: actions/checkout@v4
+          - name: Python 3
+            uses: actions/setup-python@v5
+            with:
+              python-version: '3.x'
+          - name: Install dependencies
+            run: pip install pylint -r .build/run-ci.d/requirements.txt
+          - name: Lint run-ci
+            run: pylint --disable=C0301,W0511,C0103,W0702,C0415,C0116,C0115,R0914,W0603,R0915,R0913,R0917,R0911,W0212,W0621 .build/run-ci
+          - name: Run run-ci unit tests
+            run: python .build/run-ci.d/run-ci-test.py
+          - name: Check run-ci argument parsing
+            run: python .build/run-ci --help
+
+    jenkins-validate:
+        runs-on: ubuntu-latest
+
+        steps:
+          - name: Checkout code
+            uses: actions/checkout@v4
+          - name: Python 3
+            uses: actions/setup-python@v5
+            with:
+              python-version: '3.x'
+          - name: Install dependencies
+            run: pip install pyyaml
+          - name: Helm
+            uses: azure/setup-helm@v4
+          # validate.sh runs groovy from docker when it is not on the PATH
+          - name: Validate .jenkins/
+            run: .jenkins/k8s/jenkins-test.sh

.jenkins/Jenkinsfile

@@ -352,7 +352,7 @@ def build(command, cell) {
         ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
           try {
             fetchSource(cell.step, cell.arch, cell.jdk)
-            sh """
+            sh label: "checking Jenkinsfile validity", script: """
                 test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
                 grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
               """
@@ -370,7 +370,7 @@ def build(command, cell) {
                 }
                 if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
                 if ("jar" == cell.step) {
-                  stash name: "${cell.arch}_${cell.jdk}"
+                  _stash(cell)
                 }
                 dir("build") {
                   copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
@@ -438,8 +438,8 @@ def test(command, cell) {
                   def descriptions = []
                   for (def cause in exc.getCauses()) {
                     echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
-                    if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
+                    if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption') || cause.getClass().getName().contains('ParallelStep$FailFastCause')) {
-                      throw exc // user explicitly aborted — do not retry
+                      throw exc // user abort or fail-fast — do not retry
                     }
                     descriptions.add(cause.getShortDescription())
                   }
@@ -453,24 +453,12 @@ def test(command, cell) {
               }
             }
             dir("build") {
-              sh """
+              organiseTestResultFiles(cell, cell_suffix)
-                  mkdir -p test/output/${cell.step}
-                  find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
-                  find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
-                  find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
-                """
               if (!cell.step.startsWith("microbench")) {
                 junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
               }
-              // check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
+              debugOomKiller()
-              sh """
+              compressTestResultFiles()
-                  echo "docker memory/oomkiller debug:"
-                  cat /sys/fs/cgroup/docker/memory.events || true
-                """
-              sh """
-                  find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
-                  echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
-                """
               archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true
               copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
             }
@@ -517,7 +505,7 @@ def fetchDockerImages(dockerfiles) {
   // prefetch, from apache jfrog, reduces risking dockerhub pull rate limits
   // also prefetch alpine:latest as its used as a utility in the scripts
   def dockerfilesVar = dockerfiles.join(' ')
-  sh """#!/bin/bash
+  sh label: "fetching docker images...", script: """#!/bin/bash
       for dockerfile in ${dockerfilesVar} ; do
         image_tag="\$(md5sum .build/docker/\${dockerfile}.docker | cut -d' ' -f1)"
         image_name="apache/cassandra-\${dockerfile}:\${image_tag}"
@@ -527,6 +515,14 @@ def fetchDockerImages(dockerfiles) {
       done
       docker pull -q apache.jfrog.io/cassan-docker/alpine:3.19.1 &
       wait
+      # debug how much space the images have taken.  df reports the node's filesystem: an emptyDir has no
+      # size of its own, so this sees node pressure coming, never the docker-storage sizeLimit
+      { set +x; } 2>/dev/null
+      free_gib=\$(df --output=avail -B 1073741824 / | tail -1 | tr -d ' ')
+      echo "docker images total: \$(docker system df --format '{{.Size}}' | head -1) (node free: \${free_gib}Gi)"
+      if [ "\${free_gib}" -le 10 ] ; then
+        echo "WARNING: only \${free_gib}Gi free on the node after pulling images — review the agents' ephemeral-storage budget in .jenkins/k8s/jenkins-deployment.yaml"
+      fi
      """
 }
 
@@ -564,15 +560,15 @@ def copyToNightlies(sourceFiles, remoteDirectory='') {
 def cleanAgent(job_name) {
   // get any public IP which is more helpful correlating back to the cloud instance
   sh script: 'hostname; curl -sm 10 ifconfig.me', returnStatus: true
+  def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/"
   if (isCanonical()) {
-    def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/"
     cleanAgentDocker(job_name, agentScriptsUrl)
-    logAgentInfo(job_name, agentScriptsUrl)
   }
+  logAgentInfo(job_name, agentScriptsUrl)
   cleanWs()
   if (isCanonical()) {
     // in the workspace prune any abandoned or uncleaned builds (CASSANDRA-20436)
-    sh """#!/bin/bash
+    sh label: "prune abandoned and uncleaned workspace builds files...", script: """#!/bin/bash
         set +e
         find /home/jenkins/jenkins-*/workspace/ -mindepth 2 -maxdepth 2 -type d -regextype posix-extended -regex '.*/[0-9]+' -mtime +31 -print -exec rm -rf {} +
     """
@@ -582,8 +578,7 @@ def cleanAgent(job_name) {
 def cleanAgentDocker(job_name, agentScriptsUrl) {
   // we don't expect any build to have been running for longer than maxBuildHours
   def maxBuildHours = 12
-  echo "Pruning docker for '${job_name}' on ${NODE_NAME}…" ;
+  sh label: "Pruning docker for '${job_name}' on ${NODE_NAME}...", script: """#!/bin/bash
-  sh """#!/bin/bash
       set +e
       wget -q ${agentScriptsUrl}/docker_image_pruner.py
       wget -q ${agentScriptsUrl}/docker_agent_cleaner.sh
@@ -592,12 +587,58 @@ def cleanAgentDocker(job_name, agentScriptsUrl) {
 }
 
 def logAgentInfo(job_name, agentScriptsUrl) {
-  sh """#!/bin/bash
+  // post-run remaining build/ and docker usage. used to validate the agents' ephemeral-storage budget
-      set +e -o pipefail
+  sh label: "log build usage...", script: """
-      wget -q ${agentScriptsUrl}/agent_report.sh
+      { set +x; } 2>/dev/null
-      bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt
+      du -sh ${WORKSPACE}/build/m2 ${WORKSPACE}/build/tmp ${WORKSPACE}/build/test ${WORKSPACE}/build 2>/dev/null || true
-     """
+      df -h / || true
-  copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/")
+    """
+  if (isCanonical()) {
+    sh label: "running agent_report.sh for disk usage stats (and more)...", script: """#!/bin/bash
+        set +e -o pipefail
+        wget -q ${agentScriptsUrl}/agent_report.sh
+        bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt
+      """
+    copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/")
+  }
+}
+
+def _stash(cell) {
+  sh label: "check stash size...", script: """
+      { set +x; } 2>/dev/null
+      free_gib=\$(df --output=avail -B 1073741824 / | tail -1 | tr -d ' ')
+      stash_gb=\$(du -sb ${WORKSPACE} | awk '{printf "%.1f", \$1/1024/1024/1024}')
+      echo "stash size: \${stash_gb}G (node free: \${free_gib}Gi)"
+      if [ "\${free_gib}" -le 10 ] ; then
+        echo "WARNING: only \${free_gib}Gi free on the node after building stash (\${stash_gb}G) — review jnlp's resourceLimitEphemeralStorage in .jenkins/k8s/jenkins-deployment.yaml"
+      fi
+    """
+  stash name: "${cell.arch}_${cell.jdk}"
+}
+
+def organiseTestResultFiles(cell, cell_suffix) {
+  sh label: "organise test result files...", script: """
+      mkdir -p test/output/${cell.step}
+      find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
+      find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
+      find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
+    """
+}
+
+def debugOomKiller() {
+  // check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
+  sh label: "checking for oom kills...", script: """
+      # docker memory/oomkiller debug:
+      cat /sys/fs/cgroup/docker/memory.events || true
+    """
+}
+
+def compressTestResultFiles() {
+  sh label: "compress test result files...", script: """
+      { set +x; } 2>/dev/null
+      find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
+      echo "\$(find test/output -type f -name "*.xml.xz" | wc -l) test result files compressed"
+    """
 }
 
 /////////////////////////////////////////
@@ -614,7 +655,7 @@ def generateTestReports() {
     def script_vars = "#!/bin/bash -x \n "
     if (isCanonical()) {
       // copyArtifacts takes >4hrs, hack with manual download
-      sh """${script_vars}
+      sh label: "manual download (instead of copyArtifacts)...", script: """${script_vars}
           ( mkdir -p build/test
           wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip
           unzip -x -d build/test -q output.zip ) ${teeSuffix}
@@ -627,7 +668,7 @@ def generateTestReports() {
       // merge splits for each target's test report, other axes are kept separate
       //   TODO parallelised for loop
       //   TODO results_details.tar.xz needs to include all logs for failed tests
-      sh """${script_vars} (
+      sh label: "merging splits test reports...", script: """${script_vars} (
           echo "test result files to decompress"; find build/test/output -type f -name "*.xml.xz" | wc -l
           find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f --decompress
 

.jenkins/k8s/README.md

@@ -24,14 +24,14 @@ ZONE="us-central1-c"
 gcloud container clusters create ${CLUSTER_NAME} --machine-type e2-standard-8 --disk-type=pd-ssd --num-nodes 1 --node-labels=cassandra.jenkins.controller=true --autoscaling-profile optimize-utilization --zone ${ZONE}
 
 # small resource nodes
-gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE}
+gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE}
 
 # medium resource nodes
 #  preference (by cost): n2-highcpu-8, c3-highcpu-8, n4-highcpu-8, n1-highcpu-16
-gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=100 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE}
+gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=100 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE}
 
 # large resource nodes
-gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE}
+gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE}
 
 # For each sized resource nodes, pick any machine type that fits, those listed above should work and be the most cost-effective, but this can change region to region
 # See https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/Jenkinsfile#L35-L38
@@ -64,17 +64,81 @@ kubectl exec -it svc/cassius-jenkins -c jenkins -- /bin/cat /run/secrets/additio
 
 This leaves the controller running, a single e2-standard-8 instance. All other node-pools downscale to zero.
 
-### Local-only Access
+## Upgrading an existing instance (e.g. pre-ci.cassandra.apache.org)
+
+A long-lived site like pre-ci.cassandra.apache.org may carry customisations: hostname, cloud load-balancer, storage class; that are deliberately absent from `jenkins-deployment.yaml`.
+
+Running  `helm upgrade -f jenkins-deployment.yaml` or `.build/run-ci --only-setup` will drop those customisations.
+
+Instead keep the customisations in separate overrides file, using it like
+```
+.build/run-ci --only-setup --values-override <file>
+```
+ or as a second `-f` argument to `helm upgrade`.
+
+
+##### To collect overridden values
+
+To get and diff currently deployed values against the version of `jenkins-deployment.yaml` that is deployed.
+```
+RELEASE=cassius
+NS=default
+kubectl config current-context   # confirm correct context
+DEPLOYED_COMMIT=cassandra-5.0 # the last jenkins-deployment.yaml deployed git commit sha 
+
+helm get values ${RELEASE} -n ${NS} -o yaml > /tmp/cassandra-ci-live-values.yaml
+
+git show ${DEPLOYED_COMMIT}:.jenkins/k8s/jenkins-deployment.yaml > /tmp/deployed.yaml
+
+for f in /tmp/deployed.yaml /tmp/cassandra-ci-live-values.yaml ; do
+  python3 -c 'import sys,yaml;print(yaml.safe_dump(yaml.safe_load(open(sys.argv[1])),sort_keys=True,width=10000))' ${f} > ${f}.sorted
+done
+
+diff -u /tmp/deployed.yaml.sorted /tmp/cassandra-ci-live-values.yaml.sorted
+```
+
+Copy the genuine customisations you need to keep into `~/.cassandra-ci/<site>-overrides.yaml`, keeping the full key path.
+
+For example:
+```
+controller:
+  ingress:
+    hostName: pre-ci.cassandra.apache.org      # note the capital N, `hostname` is silently ignored
+  serviceAnnotations:                          # AWS load-balancer-controller: static EIP, public subnet
+    service.beta.kubernetes.io/aws-load-balancer-name: pre-ci-apache-cassandra
+    ...
+persistence:
+  storageClass: gp2
+```
+
+Beware how Helm merges: maps are merged key by key, but lists and strings are *replaced* wholesale.  Each `agent.podTemplates.*` entry is one multi-line string, so overriding a pod template masks every repo-side change to that template.  Prefer keeping pod-template customisations out of the overrides file; where that is unavoidable, re-apply the repo's changes to the overridden copy by hand at each upgrade.
+
+
+### Rolling back a bad Helm upgrade
+
+```
+helm history ${RELEASE} -n ${NS}
+helm rollback ${RELEASE} <last-good-revision> -n ${NS} --wait --timeout 15m
+```
+
+`helm rollback` restores the previous chart *and* values, so the site's customisations come back with it.  If the release history itself is unusable, the `/tmp/cassandra-ci-live-values.yaml` from above can be used:
+```
+helm upgrade ${RELEASE} jenkins/jenkins --version ${CHART_VERSION} -n ${NS} \
+    -f /tmp/cassandra-ci-live-values.yaml --wait --timeout 15m
+```
+
+
+### Configuring Local-only Access
 
 If you want only local private access to Jenkins, do the following.
 
-Comment these lines before running `heml upgrade …`
+Comment these lines before running `helm upgrade …`
 ```
 #  serviceType: LoadBalancer
 #  ingress:
 #    enabled: "true"
 ```
-Run the heml upgrade and get the password as usual
+Run the helm upgrade and get the password as usual
 ```
 helm upgrade --install -f values.yaml cassius jenkins/jenkins --wait
 

.jenkins/k8s/jenkins-deployment.yaml

@@ -22,7 +22,11 @@
 persistence:
   enabled: true
   size: "500Gi"
-  # aws needs gp2, gke can be left commented
+  # keep the claim (jobs, credentials, build history) when the release is uninstalled.
+  # deleting it then takes a deliberate `kubectl delete pvc cassius-jenkins`
+  annotations:
+    helm.sh/resource-policy: keep
+  # aws needs gp2, gke can be left commented (add it to your override yaml)
   #storageClass: "gp2"
 controller:
   # To get URL run `kubectl describe svc cassius-jenkins | grep 'LoadBalancer Ingress'`
@@ -31,7 +35,7 @@ controller:
   targetPort: 8080
   ingress:
     enabled: "true"
-    # uncomment and set if you have a "ci-cassandra" dns entry for the jenkins controller
+    # if you have a "ci-cassandra" dns entry for the jenkins controller add the following value to your override yaml
     #hostName: ci-cassandra.<your-domain>
   customJenkinsLabels:
     - controller
@@ -69,13 +73,14 @@ controller:
     - "staticMethod org.codehaus.groovy.runtime.DefaultGroovyMethods inspect java.lang.Object"
     - "staticMethod org.codehaus.groovy.runtime.DefaultGroovyMethods max java.util.Collection"
     - "staticMethod org.codehaus.groovy.runtime.DefaultGroovyMethods putAt java.util.List java.util.List java.lang.Object"
+    - "field hudson.plugins.git.GitSCMBackwardCompatibility branch"
     - "method org.jenkinsci.plugins.workflow.steps.FlowInterruptedException getCauses"
   JCasC:
     configScripts:
       welcome-message: |
         jenkins:
           systemMessage: Welcome to Apache Cassandra
-      # we still need separate jobs because Jenkinsfile can differ (and are read before parameters are applied).
+      # Separate jobs are needed because Jenkinsfiles differ, and are read before parameters are applied.
       #  if a dev branch alters the Jenkinsfile, it will not be picked up by the job – you need to edit the job configuration
       #  see the CAUTION warning in .jenkins/Jenkinsfile
       # TODO: add new version each release branching
@@ -154,6 +159,23 @@ agent:
   node-selector:
     cassandra.jenkins.agent: true
   waitForPodSec: "180"
+  #
+  # Each template below is an opaque string to the helm chart, parsed only by the kubernetes plugin,
+  # so the chart (nor .jenkins/k8s/jenkins-test.sh) can validate it.
+  #
+  # Two traps then to pay attention to:
+  #  - a volume declared under `volumes:` is generated as `volume-N`, and the generated copy wins any
+  #    merge with a raw `yaml:` entry of the same name, so a field set only there is silently dropped.
+  #    Volumes wanting a field the plugin lacks, a sizeLimit for instance, are declared in `yaml:` alone.
+  #  - ephemeral-storage is charged to the pod: every emptyDir, the workspace included, along with the
+  #    containers' writable layers and logs, all against the containers' limits summed.  Declare no
+  #    request and the pod is BestEffort for storage, which the scheduler ignores and the node evicts
+  #    first, reporting only `request is 0`.  Each template below budgets 80Gi (of the ~89Gi that a
+  #    100GiB node allocates) leaving the rest to the node's own image cache and daemonsets.
+  #
+  # After any deploy, validate changes like:
+  #   kubectl get pod -l jenkins/cassius-jenkins-agent -o json | jq '.items[0].spec | {volumes, containers: [.containers[] | {name, resources, volumeMounts}]}'
+  #
   podTemplates:
     agent-dind-small: |
       - name: agent-dind-small
@@ -201,6 +223,9 @@ agent:
             resourceLimitCpu: 2
             resourceRequestMemory: 1G
             resourceLimitMemory: 1G
+            # the workspace emptyDir
+            resourceRequestEphemeralStorage: 10Gi
+            resourceLimitEphemeralStorage: 20Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
           - name: dind
@@ -225,12 +250,13 @@ agent:
             resourceLimitCpu: 4
             resourceRequestMemory: 1G
             resourceLimitMemory: 2400M
+            # docker's images and containers, in the docker-storage emptyDir
+            resourceRequestEphemeralStorage: 40Gi
+            resourceLimitEphemeralStorage: 60Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
         volumes:
-          - emptyDirVolume:
+          # /var/lib/docker is not here but in `yaml:` below, the only place it can carry a sizeLimit
-              memory: 'false'
-              mountPath: /var/lib/docker
           - emptyDirVolume:
               memory: 'false'
               mountPath: /certs
@@ -247,6 +273,18 @@ agent:
                           values:
                             - "true"
                     topologyKey: kubernetes.io/hostname
+            # docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge.
+            # 60Gi bounds the images and their containers alone, of the pod's 80Gi
+            # fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow.
+            volumes:
+              - name: docker-storage
+                emptyDir:
+                  sizeLimit: 60Gi
+            containers:
+              - name: dind
+                volumeMounts:
+                  - name: docker-storage
+                    mountPath: /var/lib/docker
     agent-dind-medium: |
       - name: agent-dind-medium
         label: agent-dind cassandra-medium cassandra-amd64-medium
@@ -291,6 +329,9 @@ agent:
             resourceLimitCpu: 3
             resourceRequestMemory: 1G
             resourceLimitMemory: 2400M
+            # the workspace emptyDir
+            resourceRequestEphemeralStorage: 10Gi
+            resourceLimitEphemeralStorage: 20Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
           - name: dind
@@ -315,12 +356,13 @@ agent:
             resourceLimitCpu: 4
             resourceRequestMemory: 3400M
             resourceLimitMemory: 5G
+            # docker's images and containers, in the docker-storage emptyDir
+            resourceRequestEphemeralStorage: 40Gi
+            resourceLimitEphemeralStorage: 60Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
         volumes:
-          - emptyDirVolume:
+          # /var/lib/docker is not here but in `yaml:` below, the only place it can carry a sizeLimit
-              memory: 'false'
-              mountPath: /var/lib/docker
           - emptyDirVolume:
               memory: 'false'
               mountPath: /certs
@@ -337,6 +379,18 @@ agent:
                           values:
                             - "true"
                     topologyKey: kubernetes.io/hostname
+            # docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge.
+            # 60Gi bounds the images and their containers alone, of the pod's 80Gi
+            # fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow.
+            volumes:
+              - name: docker-storage
+                emptyDir:
+                  sizeLimit: 60Gi
+            containers:
+              - name: dind
+                volumeMounts:
+                  - name: docker-storage
+                    mountPath: /var/lib/docker
     agent-dind-large: |
       - name: agent-dind-large
         label: agent-dind cassandra-large cassandra-amd64-large cassandra-amd64-large-dedicated
@@ -381,6 +435,9 @@ agent:
             resourceLimitCpu: 3
             resourceRequestMemory: 1G
             resourceLimitMemory: 2G
+            # the workspace emptyDir
+            resourceRequestEphemeralStorage: 10Gi
+            resourceLimitEphemeralStorage: 20Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
           - name: dind
@@ -405,12 +462,13 @@ agent:
             resourceLimitCpu: 7
             resourceRequestMemory: 16G
             resourceLimitMemory: 30G
+            # docker's images and containers, in the docker-storage emptyDir
+            resourceRequestEphemeralStorage: 40Gi
+            resourceLimitEphemeralStorage: 60Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
         volumes:
-          - emptyDirVolume:
+          # /var/lib/docker is not here but in `yaml:` below, the only place it can carry a sizeLimit
-              memory: 'false'
-              mountPath: /var/lib/docker
           - emptyDirVolume:
               memory: 'false'
               mountPath: /certs
@@ -427,5 +485,17 @@ agent:
                           values:
                             - "true"
                     topologyKey: kubernetes.io/hostname
+            # docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge.
+            # 60Gi bounds the images and their containers alone, of the pod's 80Gi
+            # fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow.
+            volumes:
+              - name: docker-storage
+                emptyDir:
+                  sizeLimit: 60Gi
+            containers:
+              - name: dind
+                volumeMounts:
+                  - name: docker-storage
+                    mountPath: /var/lib/docker
 
 

.jenkins/k8s/jenkins-test.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Validates the CI declarations under .jenkins/ without deploying anything:
+#  - the Jenkinsfile parses as groovy
+#  - jenkins-deployment.yaml renders through the jenkins helm chart
+#  - the yaml embedded in it (agent pod templates, JCasC config scripts) parses
+#
+# Requires: helm, python3 with pyyaml, and either groovy or docker.
+# Run from anywhere: .jenkins/k8s/jenkins-test.sh
+
+set -e
+
+CASSANDRA_DIR="$(cd "$(dirname "$0")/../.." > /dev/null && pwd)"
+JENKINS_DIR="${CASSANDRA_DIR}/.jenkins"
+status=0
+
+command -v helm > /dev/null || { echo "helm must be installed and in the PATH"; exit 1; }
+command -v python3 > /dev/null || { echo "python3 must be installed and in the PATH"; exit 1; }
+python3 -c "import yaml" 2> /dev/null || { echo "python3 pyyaml must be installed: pip install pyyaml"; exit 1; }
+
+echo "== Jenkinsfile groovy syntax"
+# Phases.CONVERSION parses and builds the AST without resolving the pipeline DSL or @NonCPS,
+# neither of which exist outside a jenkins controller
+syntax_check_dir="$(mktemp -d)"
+syntax_check="${syntax_check_dir}/syntax-check.groovy"
+# mktemp gives 0700, which the unprivileged user inside the groovy image cannot traverse
+chmod 755 "${syntax_check_dir}"
+cat > "${syntax_check}" << 'EOF'
+import org.codehaus.groovy.control.CompilationUnit
+import org.codehaus.groovy.control.Phases
+
+def cu = new CompilationUnit()
+args.each { cu.addSource(new File(it)) }
+cu.compile(Phases.CONVERSION)
+println "  ${args.join(', ')} parses"
+EOF
+if command -v groovy > /dev/null ; then
+  groovy "${syntax_check}" "${JENKINS_DIR}/Jenkinsfile" || status=1
+elif command -v docker > /dev/null ; then
+  # absolute paths, the image's working directory is not where the script was mounted
+  docker run --rm -v "${syntax_check_dir}:/check:ro" -v "${JENKINS_DIR}:/jenkins:ro" \
+      groovy:4.0-jdk17 groovy /check/syntax-check.groovy /jenkins/Jenkinsfile || status=1
+else
+  echo "  SKIPPED: neither groovy nor docker found"
+fi
+
+echo "== jenkins-deployment.yaml renders through the helm chart"
+helm repo add jenkins https://charts.jenkins.io > /dev/null
+helm repo update > /dev/null
+# --namespace and a release name only so the chart's templates have something to interpolate
+helm template cassius jenkins/jenkins --namespace default -f "${JENKINS_DIR}/k8s/jenkins-deployment.yaml" > /dev/null \
+  && echo "  jenkins-deployment.yaml renders" || status=1
+
+echo "== yaml embedded in jenkins-deployment.yaml"
+python3 - "${JENKINS_DIR}/k8s" << 'EOF' || status=1
+import sys, yaml
+from pathlib import Path
+
+k8s_dir = Path(sys.argv[1])
+errors = 0
+
+for path in sorted(k8s_dir.glob("*.yaml")):
+    try:
+        values = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    except yaml.YAMLError as error:
+        print(f"  INVALID {path.name}: {error}")
+        errors += 1
+        continue
+    print(f"  {path.name} parses")
+
+    # the values in these two maps are themselves yaml documents, and a chart never validates them —
+    # a misindented pod template reaches the kubernetes plugin and silently loses its agents
+    for keys in (("agent", "podTemplates"), ("controller", "JCasC", "configScripts")):
+        embedded = values
+        for key in keys:
+            embedded = embedded.get(key, {}) if isinstance(embedded, dict) else {}
+        for name, document in (embedded or {}).items():
+            location = f"{path.name} {'.'.join(keys)}.{name}"
+            try:
+                parsed = yaml.safe_load(document)
+            except yaml.YAMLError as error:
+                print(f"  INVALID {location}: {error}")
+                errors += 1
+                continue
+            print(f"  {location} parses")
+            # each pod template may carry a raw kubernetes pod spec in a nested `yaml` key
+            for template in parsed if isinstance(parsed, list) else []:
+                if isinstance(template, dict) and "yaml" in template:
+                    try:
+                        yaml.safe_load(template["yaml"])
+                    except yaml.YAMLError as error:
+                        print(f"  INVALID {location}.yaml: {error}")
+                        errors += 1
+                    else:
+                        print(f"  {location}.yaml parses")
+
+sys.exit(1 if errors else 0)
+EOF
+
+[ 0 -eq ${status} ] && echo "== all .jenkins/ checks passed" || echo "== FAILED"
+exit ${status}