Merge 8cab0e0af0 into 209ef2b6c4

* Add ephemeral-storage requests and limits to agents, and log peak disk usage
Agents were evicted at the node level due to lack of disk space. Each pod needs minimum 100GiB, and each template now budgets 80Gi of the ~89Gi that a 100GiB node allocates, with docker's own volume carries a 60Gi sizeLimit.  Exceeding either evicts the pod, so logging has been added in each test split before and after the run. All script blocks in Jenkinsfile are now labeled for improved readability in test runs.

* `.build/run-ci` must not silently drop a site's helm values customisations
Long-lived instances like pre-ci.cassandra.apache.org carry values in addition to the jenkins-deployment.yaml, and any `helm upgrade -f jenkins-deployment.yaml` (or `.build/run-ci`) would drop them.  Allow these values to be kept in a separate file, passed in with `--values-override`. Before upgrading helm values, we now compare the deployed values against what is about to be applied and require confirmation for anything only the site holds.  This catches added keys and dropped list items (but it cannot catch every change).
Also keep the jenkins-home pvc claim on uninstall, so --only-tear-down no longer destroys build history. Add jenkins-test.sh and a gha workflow for test validating changes under .jenkins/

 patch by Mick Semb Wever; reviewed by Dmitry Konstantinov for CASSANDRA-21523

.build/build-rat.xml

@@ -72,7 +72,7 @@
                  <exclude name="test/data/jmxdump/cassandra-*-jmx.yaml"/>
                  <!-- Documentation files -->
                  <exclude name=".github/pull_request_template.md"/>
-                 <exclude name=".github/workflows/code-check.yaml"/>
+                 <exclude name=".github/workflows/**.yaml"/>
                  <exclude NAME="doc/modules/**/*"/>
                  <exclude NAME="src/java/**/*.md"/>
                  <exclude NAME="**/README*"/>

.build/run-ci

@@ -25,7 +25,7 @@ Python dependencies are found in .build/run-ci.d/requirements.txt
 Custom environment variables can be set in .build/.run-ci.env
 
 lint with:
- `pylint --disable=C0301,W0511,C0103,W0702,C0415,C0116,C0115,R0914,W0603,R0915,R0913,R0911 run-ci`
+ `pylint --disable=C0301,W0511,C0103,W0702,C0415,C0116,C0115,R0914,W0603,R0915,R0913,R0917,R0911,W0212,W0621 run-ci`
 
 test with:
  `python .build/run-ci.d/run-ci-test.py`
@@ -38,6 +38,7 @@ import gzip
 import itertools
 import os
 import shutil
+import socket
 import subprocess
 import sys
 import tarfile
@@ -50,9 +51,10 @@ from urllib.request import urlretrieve
 from typing import Optional, Tuple
 
 # External Libraries (`pip install -r .build/run-ci.d/requirements.txt`)
+import requests
+import yaml
 from bs4 import BeautifulSoup
 from kubernetes import client, config, stream
-import requests
 
 try:
     import jenkins
@@ -72,9 +74,9 @@ def base_job_name(args) -> str:
     """
     if not hasattr(base_job_name, "_cached_result"):
         raw_url = args.repository.replace("https://github.com/", "https://raw.githubusercontent.com/").removesuffix(".git") + f"/{args.branch}/build.xml"
-        if 200 != requests.head(raw_url).status_code:
+        if 200 != requests.head(raw_url, timeout=30).status_code:
             raise ValueError(f"GitHub unavailable, or this branch has not been pushed yet: {args.repository} @ {args.branch} (or remote tracking not setup up: `git config --get branch.{args.branch}.remote` and `git config --get branch.{args.branch}.merge`)")
-        response = requests.get(raw_url)
+        response = requests.get(raw_url, timeout=30)
         response.raise_for_status()
         for line in response.text.splitlines():
             if 'property' in line and 'name="base.version"' in line:
@@ -97,7 +99,7 @@ def is_local_git_dirty(args) -> bool:
     # use base_job_name to verify the remote branch exists
     base_job_name(args)
     # check if the working directory is clean
-    clean = subprocess.run(["git", "-C", str(CASSANDRA_DIR), "diff-index", "--quiet", "HEAD", "--"]).returncode
+    clean = subprocess.run(["git", "-C", str(CASSANDRA_DIR), "diff-index", "--quiet", "HEAD", "--"], check=False).returncode
     # check if there are unpushed committed changes
     unpushed_commits = bool(subprocess.run(["git", "-C", str(CASSANDRA_DIR), "log", "@{u}..HEAD", "--name-only"],
                                            capture_output=True, text=True, check=False).stdout.strip())
@@ -190,6 +192,7 @@ def argument_parser() -> argparse.ArgumentParser:
     parser.add_argument("-k", "--dtest-branch", default=DEFAULT_DTEST_REPO_BRANCH, help="DTest repository branch.")
     parser.add_argument("-s", "--setup", action="store_true", help="Set up Jenkins before the build.")
     parser.add_argument("--only-setup", action="store_true", help="Only install Jenkins into the k8s cluster.")
+    parser.add_argument("-f", "--values-override", help="Path to an additional helm values file, applied over .jenkins/k8s/jenkins-deployment.yaml. Required when the target cluster carries site customisations, see .jenkins/k8s/README.md")
     parser.add_argument("--tear-down", action="store_true", help="Tear down Jenkins after the build.")
     parser.add_argument("--only-tear-down", action="store_true", help="Only tear down Jenkins.")
     parser.add_argument("--only-node-cleaner", action="store_true", help="Only run the node cleaner. The node cleaner scans the k8s nodes, eagerly terminating those unused.")
@@ -210,6 +213,8 @@ def parse_arguments() -> argparse.Namespace:
     assert not (args.setup and args.only_setup), "Both --setup or --only-setup cannot be specified."
     assert not (args.tear_down and args.only_tear_down), "Both --tear-down or --only-tear-down cannot be specified."
     assert not ("custom" == args.profile and not args.profile_custom_regexp), "Custom profile requires --profile-custom-regexp."
+    assert not (args.values_override and not (args.setup or args.only_setup)), "--values-override requires --setup or --only-setup."
+    assert not (args.values_override and not Path(args.values_override).is_file()), f"No such values override file: {args.values_override}"
 
     if not args.url and os.environ.get("JENKINS_URL"):
         args.url = os.environ.get("JENKINS_URL")
@@ -251,19 +256,113 @@ def run_kubectl_command(kubeconfig: Optional[str], kubecontext: Optional[str], k
     cmd += command
     return subprocess.run(cmd, capture_output=True, text=True, check=True).stdout.strip()
 
-def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ns: str):
-    """Installs Jenkins Operator using Helm in the specified K8s namespace."""
-    print("Adding Helm repository for Jenkins Operator...")
-    subprocess.run(["helm", "repo", "add", "jenkins", "https://charts.jenkins.io"], check=True)
-    subprocess.run(["helm", "repo", "update"], check=True)
-
+def run_helm_command(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ns: str, command: list,
+                     capture_output: bool = True, check: bool = True) -> subprocess.CompletedProcess:
+    """Runs a helm command with the specified kubeconfig, context and namespace."""
     cmd = ["helm"]
     if kubeconfig:
         cmd += ["--kubeconfig", kubeconfig]
     if kubecontext:
         cmd += ["--kube-context", kubecontext]
-    cmd += ["--namespace", kube_ns, "upgrade", "--install", "-f", DEPLOY_YAML, "cassius", "jenkins/jenkins", "--wait"]
-    result = subprocess.run(cmd, capture_output=True, check=True)
+    cmd += ["--namespace", kube_ns]
+    cmd += command
+    return subprocess.run(cmd, capture_output=capture_output, text=True, check=check)
+
+def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ns: str,
+                    values_override: Optional[str] = None):
+    """Installs Jenkins Operator using Helm in the specified K8s namespace."""
+
+    def confirm_helm_updates():
+        """Prompts before an upgrade drops any values the deployed jenkins currently has."""
+
+        def helm_values() -> dict:
+            """
+            The values a deployed jenkins was last installed with, or an empty dict when there is no release.
+            These are the user-supplied values, whatever files they came from, so they include any customisations
+            made to the site outside of `.jenkins/k8s/jenkins-deployment.yaml`.
+            """
+            result = run_helm_command(kubeconfig, kubecontext, kube_ns,
+                                      ["get", "values", "cassius", "-o", "yaml"], check=False)
+            if result.returncode != 0:
+                debug(f"No existing cassius release found in namespace {kube_ns}: {result.stderr.strip()}")
+                return {}
+            return yaml.safe_load(result.stdout) or {}
+
+        def merge_values(base: dict, override: dict) -> dict:
+            """Merges two helm values files the way helm does: maps key by key, everything else replaced."""
+            merged = dict(base)
+            for key, value in override.items():
+                if isinstance(value, dict) and isinstance(merged.get(key), dict):
+                    merged[key] = merge_values(merged[key], value)
+                else:
+                    merged[key] = value
+            return merged
+
+        def detect_lost_values(live: dict, proposed: dict) -> dict:
+            """
+            Values the deployed jenkins has that an upgrade would drop, as {dotted.key.path: live value}.
+
+            A key held live but absent from what is about to be applied is either a customisation made to this site,
+            or a key that `.jenkins/k8s/jenkins-deployment.yaml` has removed since the site was last deployed.
+            Note also what this cannot see: a key that exists in both but was given a different value locally, such
+            as an edited `agent.podTemplates` entry, is silently overwritten.
+            Read the diff of `helm template` before deploying an unfamiliar site.
+            """
+            def leaf_values(values, path: str = "") -> dict:
+                """Flattens a values map to {dotted.key.path: value}, lists are leaves (helm replaces them)."""
+                if not isinstance(values, dict):
+                    return {path: values}
+                leaves = {}
+                for key, value in values.items():
+                    leaves.update(leaf_values(value, f"{path}.{key}" if path else str(key)))
+                return leaves
+
+            live_leaves, proposed_leaves = leaf_values(live), leaf_values(proposed)
+            lost = {path: value for path, value in live_leaves.items() if path not in proposed_leaves}
+            # lists are replaced wholesale, so also report items dropped from a list that is otherwise still there
+            for path, value in live_leaves.items():
+                if isinstance(value, list) and isinstance(proposed_leaves.get(path), list):
+                    dropped = [item for item in value if item not in proposed_leaves[path]]
+                    if dropped:
+                        lost[f"{path}[]"] = dropped
+            return lost
+
+        with open(DEPLOY_YAML, encoding="utf-8") as deploy_yaml:
+            proposed = yaml.safe_load(deploy_yaml) or {}
+        if values_override:
+            with open(values_override, encoding="utf-8") as override_yaml:
+                proposed = merge_values(proposed, yaml.safe_load(override_yaml) or {})
+
+        lost = detect_lost_values(helm_values(), proposed)
+        if not lost:
+            return
+
+        print(f"\nWARNING: {len(lost)} value(s) the deployed jenkins has are absent from what is about to be applied.")
+        print("Each is either a customisation of this site, or a key removed from jenkins-deployment.yaml since"
+              " the site was last deployed.  Upgrading drops them:\n")
+        for path in sorted(lost):
+            value = str(lost[path]).replace("\n", " ")
+            print(f"  {path}: {value[:100] + '…' if len(value) > 100 else value}")
+        print(f"\nTo keep any of them, add them to a values override file (see .jenkins/k8s/README.md) and pass"
+              f" `--values-override`{' (the file passed does not contain them)' if values_override else ''}.")
+
+        if not sys.stdin.isatty():
+            print("Refusing to drop them when running non-interactively.")
+            sys.exit(1)
+        if input("\nDrop these values and continue? [y/N] ").strip().lower() not in ("y", "yes"):
+            print("Aborted, nothing was deployed.")
+            sys.exit(1)
+
+    confirm_helm_updates()
+
+    print("Adding Helm repository for Jenkins Operator...")
+    subprocess.run(["helm", "repo", "add", "jenkins", "https://charts.jenkins.io"], check=True)
+    subprocess.run(["helm", "repo", "update"], check=True)
+
+    # site customisations are applied last, helm merges each -f over the previous
+    values_files = ["-f", DEPLOY_YAML] + (["-f", values_override] if values_override else [])
+    result = run_helm_command(kubeconfig, kubecontext, kube_ns,
+                              ["upgrade", "--install"] + values_files + ["cassius", "jenkins/jenkins", "--wait"])
 
     run_kubectl_command(kubeconfig, kubecontext, kube_ns,
                         ["exec", DEFAULT_POD_NAME, "--",
@@ -275,6 +374,19 @@ def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_
         sys.exit(1)
 
 
+def wait_for_jenkins_http(ip: str):
+    host, port = (ip.rsplit(":", 1)[0], int(ip.rsplit(":", 1)[1])) if ":" in ip else (ip, 80)
+    spin_while(f"Waiting for Jenkins HTTP at {host}:{port}… ", lambda: _tcp_connect_ok(host, port))
+
+
+def _tcp_connect_ok(host: str, port: int) -> bool:
+    try:
+        with socket.create_connection((host, port), timeout=2):
+            return True
+    except OSError:
+        return False
+
+
 def get_jenkins(k8s_client: client.CoreV1Api, args, kube_ns: str) -> Tuple[str, jenkins.Jenkins]:
     """Authenticates to Jenkins and returns the Jenkins ip and server objects."""
 
@@ -780,13 +892,10 @@ def cleanup_and_maybe_teardown(kubeconfig: Optional[str], kubecontext: Optional[
     IS_RUNNING = False
     if tear_down:
         print("Cleaning up Jenkins and all resources.")
-        cmd = ["helm"]
-        if kubeconfig:
-            cmd += ["--kubeconfig", kubeconfig]
-        if kubecontext:
-            cmd += ["--kube-context", kubecontext]
-        cmd += ["--namespace", kube_ns, "uninstall", "cassius"]
-        subprocess.run(cmd, check=True)
+        run_helm_command(kubeconfig, kubecontext, kube_ns, ["uninstall", "cassius"], capture_output=False)
+        # the pvc is annotated `helm.sh/resource-policy: keep`, see .jenkins/k8s/jenkins-deployment.yaml
+        print(f"Jenkins uninstalled. The jenkins-home volume was kept, delete it with:\n"
+              f"  kubectl --namespace {kube_ns} delete pvc cassius-jenkins")
 
 
 @contextmanager
@@ -824,10 +933,11 @@ def main():
     if args.setup or args.only_setup:
         init_k8s_namespace(k8s_client, DEFAULT_KUBE_NS)
         with helm_installation_lock(Path("/tmp/.cassandra-run-ci.lock")):
-            install_jenkins(args.kubeconfig, args.kubecontext, DEFAULT_KUBE_NS)
+            install_jenkins(args.kubeconfig, args.kubecontext, DEFAULT_KUBE_NS, args.values_override)
 
     (ip, server) = get_jenkins(k8s_client, args, DEFAULT_KUBE_NS)
     if args.setup or args.only_setup:
+        wait_for_jenkins_http(ip)
         ensure_cassandra_job_parameters_visible(server)
     if args.only_setup:
         return

.build/run-ci.d/README.md

@@ -3,7 +3,7 @@
 ```
 ➤ .build/run-ci --help
 usage: run-ci [-h] [-c KUBECONFIG] [-x KUBECONTEXT] [-i URL] [-u USER] [-r REPOSITORY] [-b BRANCH] [-p {packaging,skinny,pre-commit,pre-commit w/ upgrades,post-commit,custom}] [-e PROFILE_CUSTOM_REGEXP] [-j JDK] [-d DTEST_REPOSITORY] [-k DTEST_BRANCH]
-              [-s] [--only-setup] [--tear-down] [--only-tear-down] [--only-node-cleaner] [-o DOWNLOAD_RESULTS]
+              [-s] [--only-setup] [-v VALUES_OVERRIDE] [--tear-down] [--only-tear-down] [--only-node-cleaner] [-o DOWNLOAD_RESULTS]
 
 Run CI pipeline for Cassandra on K8s using Jenkins.
 
@@ -30,6 +30,8 @@ options:
                         DTest repository branch.
   -s, --setup           Set up Jenkins before the build.
   --only-setup          Only install Jenkins into the k8s cluster.
+  -v VALUES_OVERRIDE, --values-override VALUES_OVERRIDE
+                        Path to an additional helm values file, applied over .jenkins/k8s/jenkins-deployment.yaml. Required when the target cluster carries site customisations, see .jenkins/k8s/README.md
   --tear-down           Tear down Jenkins after the build.
   --only-tear-down      Only tear down Jenkins.
   --only-node-cleaner   Only run the node cleaner. The node cleaner scans the k8s nodes, eagerly terminating those unused.
@@ -63,7 +65,15 @@ Setup/Update Jenkins Helm into your current kubeconfig
 .build/run-ci --only-setup
 ```
 
-Uninstall Jenkins from your current kubeconfig
+Setup/Update Jenkins Helm into a cluster that carries site customisations, e.g. pre-ci.cassandra.apache.org
+```
+.build/run-ci --only-setup --values-override ~/.cassandra-ci/pre-ci-overrides.yaml
+```
+
+Before any setup, the values already deployed are compared against those about to be applied.  Any value the deployed jenkins holds that the new files lack is listed, and confirmation is asked for before it is dropped; running non-interactively aborts instead.  See `.jenkins/k8s/README.md` for what this can and cannot catch.
+
+Uninstall Jenkins from your current kubeconfig.
 ```
 .build/run-ci --only-tear-down
 ```
+The jenkins-home volume is kept; delete it separately with `kubectl delete pvc cassius-jenkins`

.build/run-ci.d/requirements.txt

@@ -18,6 +18,7 @@ bs4
 dotenv
 kubernetes
 python-jenkins
+pyyaml
 requests
 
 # optional for different clouds

.build/run-ci.d/run-ci-test.py

@@ -27,12 +27,14 @@
 
 import argparse
 from pathlib import Path
+import tempfile
 import unittest
 from unittest.mock import patch, MagicMock
 
 
 # Import the functions from the script
 from run_ci import (
+    DEPLOY_YAML,
     debug,
     install_jenkins,
     get_jenkins,
@@ -55,13 +57,84 @@ class TestCIPipeline(unittest.TestCase):
         debug("Test message")
         mock_print.assert_called_with("Test message")
 
+    # the pre-flight check is nested inside install_jenkins, so it is exercised through it: the mocked
+    # `helm get values` stdout stands in for the values the site already has deployed
+    LIVE_STORAGE_CLASS = "persistence:\n  storageClass: gp2\n"
+
     @patch('run_ci.subprocess.run')
     def test_install_jenkins(self, mock_run):
-        mock_run.return_value = MagicMock(returncode=0)
+        # empty stdout, i.e. nothing deployed yet, so the pre-flight check has nothing to warn about
+        mock_run.return_value = MagicMock(returncode=0, stdout="")
         install_jenkins("test-namespace", Path("/fake/cassandra/dir"), "default")
         mock_run.assert_any_call(["helm", "repo", "add", "jenkins", "https://charts.jenkins.io"], check=True)
         mock_run.assert_any_call(["helm", "repo", "update"], check=True)
 
+    @patch('run_ci.subprocess.run')
+    def test_install_jenkins_values_override(self, mock_run):
+        mock_run.return_value = MagicMock(returncode=0, stdout="")
+        with tempfile.NamedTemporaryFile("w", suffix=".yaml") as override:
+            override.write(self.LIVE_STORAGE_CLASS)
+            override.flush()
+            install_jenkins(None, None, "default", override.name)
+            upgrade_cmd = [c for c in [call.args[0] for call in mock_run.call_args_list] if "upgrade" in c][0]
+            # the site's overrides must come after, and never replace, the repo's deployment yaml
+            self.assertEqual(["-f", DEPLOY_YAML, "-f", override.name], upgrade_cmd[5:9])
+
+    @patch('run_ci.sys.stdin.isatty')
+    @patch('run_ci.print')
+    @patch('run_ci.subprocess.run')
+    def test_install_jenkins_aborts_non_interactively(self, mock_run, mock_print, mock_isatty):
+        mock_isatty.return_value = False
+        mock_run.return_value = MagicMock(returncode=0, stdout=self.LIVE_STORAGE_CLASS)
+        with self.assertRaises(SystemExit):
+            install_jenkins(None, None, "default")
+        self.assertEqual([], [c for c in [call.args[0] for call in mock_run.call_args_list] if "upgrade" in c])
+        # and continues when the customisation is passed back in as an override.  This also proves the merge is
+        # per key: were the override to replace the whole persistence map, its other keys would now be reported lost
+        with tempfile.NamedTemporaryFile("w", suffix=".yaml") as override:
+            override.write(self.LIVE_STORAGE_CLASS)
+            override.flush()
+            install_jenkins(None, None, "default", override.name)
+        self.assertEqual(1, len([c for c in [call.args[0] for call in mock_run.call_args_list] if "upgrade" in c]))
+
+    @patch('run_ci.input')
+    @patch('run_ci.sys.stdin.isatty')
+    @patch('run_ci.print')
+    @patch('run_ci.subprocess.run')
+    def test_install_jenkins_prompts(self, mock_run, mock_print, mock_isatty, mock_input):
+        mock_isatty.return_value = True
+        mock_run.return_value = MagicMock(returncode=0, stdout=self.LIVE_STORAGE_CLASS)
+        mock_input.return_value = "n"
+        with self.assertRaises(SystemExit):
+            install_jenkins(None, None, "default")
+        mock_input.return_value = "y"
+        install_jenkins(None, None, "default")
+
+    @patch('run_ci.sys.stdin.isatty')
+    @patch('run_ci.print')
+    @patch('run_ci.subprocess.run')
+    def test_install_jenkins_reports_only_detectable_losses(self, mock_run, mock_print, mock_isatty):
+        mock_isatty.return_value = False
+        # a plugin only this site installs is reported, as is a key the site alone holds; a key held in both but
+        # locally edited (persistence.size) cannot be seen, and must not be claimed
+        mock_run.return_value = MagicMock(returncode=0, stdout="persistence:\n  size: 1Ti\n"
+                                          "controller:\n  installPlugins:\n    - site-only-plugin\n")
+        with self.assertRaises(SystemExit):
+            install_jenkins(None, None, "default")
+        printed = " ".join(str(call.args[0]) for call in mock_print.call_args_list if call.args)
+        self.assertIn("controller.installPlugins[]", printed)
+        self.assertIn("site-only-plugin", printed)
+        self.assertNotIn("persistence.size", printed)
+
+    @patch('run_ci.print')
+    @patch('run_ci.subprocess.run')
+    def test_install_jenkins_when_nothing_deployed(self, mock_run, mock_print):
+        # `helm get values` fails when there is no release, and nothing is then warned about
+        mock_run.side_effect = lambda cmd, **kwargs: MagicMock(returncode=1 if "get" in cmd else 0,
+                                                              stdout="", stderr="release: not found")
+        install_jenkins(None, None, "default")
+        self.assertEqual([], [call.args[0] for call in mock_print.call_args_list if "WARNING" in str(call.args)])
+
     @patch('run_ci.subprocess.run')
     @patch('run_ci.jenkins.Jenkins')
     def test_get_jenkins(self, mock_jenkins, mock_run):
@@ -91,13 +164,14 @@ class TestCIPipeline(unittest.TestCase):
     @patch('run_ci.stream.stream')
     def test_delete_remote_junit_files(self, mock_stream):
         mock_k8s_client = MagicMock()
-        delete_remote_junit_files(mock_k8s_client, "test-pod", "test-namespace", 456)
+        delete_remote_junit_files(mock_k8s_client, "test-pod", "test-namespace", "test-job", 456)
         mock_stream.assert_called()
 
     @patch('run_ci.subprocess.run')
     def test_cleanup_and_maybe_teardown(self, mock_run):
         cleanup_and_maybe_teardown(None, None, "test-namespace", True)
-        mock_run.assert_called_with(["helm", "--namespace", "test-namespace", "uninstall", "cassius"], check=True)
+        mock_run.assert_called_with(["helm", "--namespace", "test-namespace", "uninstall", "cassius"],
+                                    capture_output=False, text=True, check=True)
 
     @patch('run_ci.fcntl.flock')
     def test_helm_installation_lock(self, mock_flock):

.circleci/config.yml

@@ -24,7 +24,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -111,10 +111,10 @@ jobs:
   j17_dtests:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -220,7 +220,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 1
+    parallelism: 10
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -402,7 +402,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -489,10 +489,10 @@ jobs:
   j11_dtests_latest_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -690,7 +690,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -804,10 +804,10 @@ jobs:
   j17_cqlsh_dtests_py38_vnode:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -912,10 +912,10 @@ jobs:
   j17_dtests_vnode_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -1066,10 +1066,10 @@ jobs:
   j11_dtests_vnode_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -1199,10 +1199,10 @@ jobs:
   j17_cqlsh_dtests_py311_latest:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -1310,7 +1310,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -1428,7 +1428,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -1518,7 +1518,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -1632,10 +1632,10 @@ jobs:
   j11_cqlsh_dtests_py38_latest:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -1744,7 +1744,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -1830,10 +1830,10 @@ jobs:
   j11_cqlsh_dtests_py311:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -1939,10 +1939,10 @@ jobs:
   j17_dtests_large_vnode_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -2071,10 +2071,10 @@ jobs:
   j17_dtests_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -2228,7 +2228,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -2346,7 +2346,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -2432,10 +2432,10 @@ jobs:
   j17_cqlsh_dtests_py311:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -2540,10 +2540,10 @@ jobs:
   j11_cqlsh_dtests_py38:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -2652,7 +2652,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -2741,7 +2741,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -2831,7 +2831,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -2948,7 +2948,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -3141,7 +3141,7 @@ jobs:
   j11_dtests_large_vnode:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: xlarge
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
     parallelism: 4
@@ -3226,10 +3226,10 @@ jobs:
   j11_dtests_large_vnode_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -3359,7 +3359,7 @@ jobs:
   j11_dtests_large:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: xlarge
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
     parallelism: 4
@@ -3447,7 +3447,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -3537,7 +3537,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -3652,10 +3652,10 @@ jobs:
   j11_upgrade_dtests_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: xlarge
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -3788,7 +3788,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 1
+    parallelism: 10
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -3905,7 +3905,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -4206,7 +4206,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -4324,7 +4324,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -4413,7 +4413,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 1
+    parallelism: 10
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -4602,7 +4602,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -4692,7 +4692,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -4957,7 +4957,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -5160,7 +5160,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 1
+    parallelism: 4
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -5350,7 +5350,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -5437,10 +5437,10 @@ jobs:
   j17_dtests_vnode:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -5543,10 +5543,10 @@ jobs:
   j11_dtests_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -5679,7 +5679,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -5797,7 +5797,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -5912,10 +5912,10 @@ jobs:
   j17_dtests_latest_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -6047,7 +6047,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -6134,10 +6134,10 @@ jobs:
   j11_upgrade_dtests:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -6219,10 +6219,10 @@ jobs:
   j11_dtests_large_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -6355,7 +6355,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -6512,7 +6512,7 @@ jobs:
   j17_dtests_large_vnode:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: xlarge
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
     parallelism: 4
@@ -6596,10 +6596,10 @@ jobs:
   j11_cqlsh_dtests_py38_vnode:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -6708,7 +6708,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -6798,7 +6798,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -6885,10 +6885,10 @@ jobs:
   j11_cqlsh_dtests_py311_latest:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -7057,10 +7057,10 @@ jobs:
   j11_cqlsh_dtests_py311_vnode:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -7169,7 +7169,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -7283,10 +7283,10 @@ jobs:
   j17_cqlsh_dtests_py38_latest:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -7391,10 +7391,10 @@ jobs:
   j17_cqlsh_dtests_py311_vnode:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -7502,7 +7502,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 1
+    parallelism: 10
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -7617,10 +7617,10 @@ jobs:
   j17_dtests_latest:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -7726,7 +7726,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -8021,7 +8021,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -8108,10 +8108,10 @@ jobs:
   j11_dtests_latest:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -8257,10 +8257,10 @@ jobs:
   j11_dtests:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -8342,10 +8342,10 @@ jobs:
   j17_cqlsh_dtests_py38:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -8453,7 +8453,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -8570,7 +8570,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -8656,7 +8656,7 @@ jobs:
   j17_dtests_large:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: xlarge
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
     parallelism: 4
@@ -8743,7 +8743,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -8904,7 +8904,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -8990,10 +8990,10 @@ jobs:
   j11_dtests_vnode:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -9078,7 +9078,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -9164,10 +9164,10 @@ jobs:
   j17_dtests_large_repeat:
     docker:
     - image: apache/cassandra-testing-ubuntu2004-java11:latest
-    resource_class: medium
+    resource_class: large
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -9299,7 +9299,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra
@@ -9388,7 +9388,7 @@ jobs:
     resource_class: medium
     working_directory: ~/
     shell: /bin/bash -eo pipefail -l
-    parallelism: 4
+    parallelism: 25
     steps:
     - attach_workspace:
         at: /home/cassandra

.github/workflows/jenkins-check.yaml

@@ -0,0 +1,58 @@
+name: CI Check
+
+# Checks the CI tooling itself: the run-ci script and everything under .jenkins/
+# Nothing here touches a k8s cluster; the helm and groovy steps only render and parse.
+
+on:
+    push:
+        branches:
+            - '**'
+        paths:
+            - '.build/run-ci'
+            - '.build/run-ci.d/**'
+            - '.jenkins/**'
+            - '.github/workflows/jenkins-check.yaml'
+    pull_request:
+        paths:
+            - '.build/run-ci'
+            - '.build/run-ci.d/**'
+            - '.jenkins/**'
+            - '.github/workflows/jenkins-check.yaml'
+
+jobs:
+    run-ci:
+        runs-on: ubuntu-latest
+
+        steps:
+          - name: Checkout code
+            uses: actions/checkout@v4
+          - name: Python 3
+            uses: actions/setup-python@v5
+            with:
+              python-version: '3.x'
+          - name: Install dependencies
+            run: pip install pylint -r .build/run-ci.d/requirements.txt
+          - name: Lint run-ci
+            run: pylint --disable=C0301,W0511,C0103,W0702,C0415,C0116,C0115,R0914,W0603,R0915,R0913,R0917,R0911,W0212,W0621 .build/run-ci
+          - name: Run run-ci unit tests
+            run: python .build/run-ci.d/run-ci-test.py
+          - name: Check run-ci argument parsing
+            run: python .build/run-ci --help
+
+    jenkins-validate:
+        runs-on: ubuntu-latest
+
+        steps:
+          - name: Checkout code
+            uses: actions/checkout@v4
+          - name: Python 3
+            uses: actions/setup-python@v5
+            with:
+              python-version: '3.x'
+          - name: Install dependencies
+            run: pip install pyyaml
+          - name: Helm
+            uses: azure/setup-helm@v4
+          # validate.sh runs groovy from docker when it is not on the PATH
+          - name: Validate .jenkins/
+            run: .jenkins/k8s/jenkins-test.sh

.jenkins/Jenkinsfile

@@ -352,7 +352,7 @@ def build(command, cell) {
         ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
           try {
             fetchSource(cell.step, cell.arch, cell.jdk)
-            sh """
+            sh label: "checking Jenkinsfile validity", script: """
                 test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
                 grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
               """
@@ -370,7 +370,7 @@ def build(command, cell) {
                 }
                 if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
                 if ("jar" == cell.step) {
-                  stash name: "${cell.arch}_${cell.jdk}"
+                  _stash(cell)
                 }
                 dir("build") {
                   copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
@@ -438,8 +438,8 @@ def test(command, cell) {
                   def descriptions = []
                   for (def cause in exc.getCauses()) {
                     echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
-                    if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
-                      throw exc // user explicitly aborted — do not retry
+                    if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption') || cause.getClass().getName().contains('ParallelStep$FailFastCause')) {
+                      throw exc // user abort or fail-fast — do not retry
                     }
                     descriptions.add(cause.getShortDescription())
                   }
@@ -453,24 +453,12 @@ def test(command, cell) {
               }
             }
             dir("build") {
-              sh """
-                  mkdir -p test/output/${cell.step}
-                  find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
-                  find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
-                  find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
-                """
+              organiseTestResultFiles(cell, cell_suffix)
               if (!cell.step.startsWith("microbench")) {
                 junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
               }
-              // check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
-              sh """
-                  echo "docker memory/oomkiller debug:"
-                  cat /sys/fs/cgroup/docker/memory.events || true
-                """
-              sh """
-                  find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
-                  echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
-                """
+              debugOomKiller()
+              compressTestResultFiles()
               archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true
               copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
             }
@@ -517,7 +505,7 @@ def fetchDockerImages(dockerfiles) {
   // prefetch, from apache jfrog, reduces risking dockerhub pull rate limits
   // also prefetch alpine:latest as its used as a utility in the scripts
   def dockerfilesVar = dockerfiles.join(' ')
-  sh """#!/bin/bash
+  sh label: "fetching docker images...", script: """#!/bin/bash
       for dockerfile in ${dockerfilesVar} ; do
         image_tag="\$(md5sum .build/docker/\${dockerfile}.docker | cut -d' ' -f1)"
         image_name="apache/cassandra-\${dockerfile}:\${image_tag}"
@@ -527,6 +515,14 @@ def fetchDockerImages(dockerfiles) {
       done
       docker pull -q apache.jfrog.io/cassan-docker/alpine:3.19.1 &
       wait
+      # debug how much space the images have taken.  df reports the node's filesystem: an emptyDir has no
+      # size of its own, so this sees node pressure coming, never the docker-storage sizeLimit
+      { set +x; } 2>/dev/null
+      free_gib=\$(df --output=avail -B 1073741824 / | tail -1 | tr -d ' ')
+      echo "docker images total: \$(docker system df --format '{{.Size}}' | head -1) (node free: \${free_gib}Gi)"
+      if [ "\${free_gib}" -le 10 ] ; then
+        echo "WARNING: only \${free_gib}Gi free on the node after pulling images — review the agents' ephemeral-storage budget in .jenkins/k8s/jenkins-deployment.yaml"
+      fi
      """
 }
 
@@ -564,15 +560,15 @@ def copyToNightlies(sourceFiles, remoteDirectory='') {
 def cleanAgent(job_name) {
   // get any public IP which is more helpful correlating back to the cloud instance
   sh script: 'hostname; curl -sm 10 ifconfig.me', returnStatus: true
+  def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/"
   if (isCanonical()) {
-    def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/"
     cleanAgentDocker(job_name, agentScriptsUrl)
-    logAgentInfo(job_name, agentScriptsUrl)
   }
+  logAgentInfo(job_name, agentScriptsUrl)
   cleanWs()
   if (isCanonical()) {
     // in the workspace prune any abandoned or uncleaned builds (CASSANDRA-20436)
-    sh """#!/bin/bash
+    sh label: "prune abandoned and uncleaned workspace builds files...", script: """#!/bin/bash
         set +e
         find /home/jenkins/jenkins-*/workspace/ -mindepth 2 -maxdepth 2 -type d -regextype posix-extended -regex '.*/[0-9]+' -mtime +31 -print -exec rm -rf {} +
     """
@@ -582,8 +578,7 @@ def cleanAgent(job_name) {
 def cleanAgentDocker(job_name, agentScriptsUrl) {
   // we don't expect any build to have been running for longer than maxBuildHours
   def maxBuildHours = 12
-  echo "Pruning docker for '${job_name}' on ${NODE_NAME}…" ;
-  sh """#!/bin/bash
+  sh label: "Pruning docker for '${job_name}' on ${NODE_NAME}...", script: """#!/bin/bash
       set +e
       wget -q ${agentScriptsUrl}/docker_image_pruner.py
       wget -q ${agentScriptsUrl}/docker_agent_cleaner.sh
@@ -592,12 +587,58 @@ def cleanAgentDocker(job_name, agentScriptsUrl) {
 }
 
 def logAgentInfo(job_name, agentScriptsUrl) {
-  sh """#!/bin/bash
-      set +e -o pipefail
-      wget -q ${agentScriptsUrl}/agent_report.sh
-      bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt
-     """
-  copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/")
+  // post-run remaining build/ and docker usage. used to validate the agents' ephemeral-storage budget
+  sh label: "log build usage...", script: """
+      { set +x; } 2>/dev/null
+      du -sh ${WORKSPACE}/build/m2 ${WORKSPACE}/build/tmp ${WORKSPACE}/build/test ${WORKSPACE}/build 2>/dev/null || true
+      df -h / || true
+    """
+  if (isCanonical()) {
+    sh label: "running agent_report.sh for disk usage stats (and more)...", script: """#!/bin/bash
+        set +e -o pipefail
+        wget -q ${agentScriptsUrl}/agent_report.sh
+        bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt
+      """
+    copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/")
+  }
+}
+
+def _stash(cell) {
+  sh label: "check stash size...", script: """
+      { set +x; } 2>/dev/null
+      free_gib=\$(df --output=avail -B 1073741824 / | tail -1 | tr -d ' ')
+      stash_gb=\$(du -sb ${WORKSPACE} | awk '{printf "%.1f", \$1/1024/1024/1024}')
+      echo "stash size: \${stash_gb}G (node free: \${free_gib}Gi)"
+      if [ "\${free_gib}" -le 10 ] ; then
+        echo "WARNING: only \${free_gib}Gi free on the node after building stash (\${stash_gb}G) — review jnlp's resourceLimitEphemeralStorage in .jenkins/k8s/jenkins-deployment.yaml"
+      fi
+    """
+  stash name: "${cell.arch}_${cell.jdk}"
+}
+
+def organiseTestResultFiles(cell, cell_suffix) {
+  sh label: "organise test result files...", script: """
+      mkdir -p test/output/${cell.step}
+      find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
+      find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
+      find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
+    """
+}
+
+def debugOomKiller() {
+  // check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
+  sh label: "checking for oom kills...", script: """
+      # docker memory/oomkiller debug:
+      cat /sys/fs/cgroup/docker/memory.events || true
+    """
+}
+
+def compressTestResultFiles() {
+  sh label: "compress test result files...", script: """
+      { set +x; } 2>/dev/null
+      find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
+      echo "\$(find test/output -type f -name "*.xml.xz" | wc -l) test result files compressed"
+    """
 }
 
 /////////////////////////////////////////
@@ -614,7 +655,7 @@ def generateTestReports() {
     def script_vars = "#!/bin/bash -x \n "
     if (isCanonical()) {
       // copyArtifacts takes >4hrs, hack with manual download
-      sh """${script_vars}
+      sh label: "manual download (instead of copyArtifacts)...", script: """${script_vars}
           ( mkdir -p build/test
           wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip
           unzip -x -d build/test -q output.zip ) ${teeSuffix}
@@ -627,7 +668,7 @@ def generateTestReports() {
       // merge splits for each target's test report, other axes are kept separate
       //   TODO parallelised for loop
       //   TODO results_details.tar.xz needs to include all logs for failed tests
-      sh """${script_vars} (
+      sh label: "merging splits test reports...", script: """${script_vars} (
           echo "test result files to decompress"; find build/test/output -type f -name "*.xml.xz" | wc -l
           find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f --decompress
 

.jenkins/k8s/README.md

@@ -24,14 +24,14 @@ ZONE="us-central1-c"
 gcloud container clusters create ${CLUSTER_NAME} --machine-type e2-standard-8 --disk-type=pd-ssd --num-nodes 1 --node-labels=cassandra.jenkins.controller=true --autoscaling-profile optimize-utilization --zone ${ZONE}
 
 # small resource nodes
-gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE}
+gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE}
 
 # medium resource nodes
 #  preference (by cost): n2-highcpu-8, c3-highcpu-8, n4-highcpu-8, n1-highcpu-16
-gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=100 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE}
+gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=100 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE}
 
 # large resource nodes
-gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE}
+gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE}
 
 # For each sized resource nodes, pick any machine type that fits, those listed above should work and be the most cost-effective, but this can change region to region
 # See https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/Jenkinsfile#L35-L38
@@ -64,17 +64,81 @@ kubectl exec -it svc/cassius-jenkins -c jenkins -- /bin/cat /run/secrets/additio
 
 This leaves the controller running, a single e2-standard-8 instance. All other node-pools downscale to zero.
 
-### Local-only Access
+## Upgrading an existing instance (e.g. pre-ci.cassandra.apache.org)
+
+A long-lived site like pre-ci.cassandra.apache.org may carry customisations: hostname, cloud load-balancer, storage class; that are deliberately absent from `jenkins-deployment.yaml`.
+
+Running  `helm upgrade -f jenkins-deployment.yaml` or `.build/run-ci --only-setup` will drop those customisations.
+
+Instead keep the customisations in separate overrides file, using it like
+```
+.build/run-ci --only-setup --values-override <file>
+```
+ or as a second `-f` argument to `helm upgrade`.
+
+
+##### To collect overridden values
+
+To get and diff currently deployed values against the version of `jenkins-deployment.yaml` that is deployed.
+```
+RELEASE=cassius
+NS=default
+kubectl config current-context   # confirm correct context
+DEPLOYED_COMMIT=cassandra-5.0 # the last jenkins-deployment.yaml deployed git commit sha 
+
+helm get values ${RELEASE} -n ${NS} -o yaml > /tmp/cassandra-ci-live-values.yaml
+
+git show ${DEPLOYED_COMMIT}:.jenkins/k8s/jenkins-deployment.yaml > /tmp/deployed.yaml
+
+for f in /tmp/deployed.yaml /tmp/cassandra-ci-live-values.yaml ; do
+  python3 -c 'import sys,yaml;print(yaml.safe_dump(yaml.safe_load(open(sys.argv[1])),sort_keys=True,width=10000))' ${f} > ${f}.sorted
+done
+
+diff -u /tmp/deployed.yaml.sorted /tmp/cassandra-ci-live-values.yaml.sorted
+```
+
+Copy the genuine customisations you need to keep into `~/.cassandra-ci/<site>-overrides.yaml`, keeping the full key path.
+
+For example:
+```
+controller:
+  ingress:
+    hostName: pre-ci.cassandra.apache.org      # note the capital N, `hostname` is silently ignored
+  serviceAnnotations:                          # AWS load-balancer-controller: static EIP, public subnet
+    service.beta.kubernetes.io/aws-load-balancer-name: pre-ci-apache-cassandra
+    ...
+persistence:
+  storageClass: gp2
+```
+
+Beware how Helm merges: maps are merged key by key, but lists and strings are *replaced* wholesale.  Each `agent.podTemplates.*` entry is one multi-line string, so overriding a pod template masks every repo-side change to that template.  Prefer keeping pod-template customisations out of the overrides file; where that is unavoidable, re-apply the repo's changes to the overridden copy by hand at each upgrade.
+
+
+### Rolling back a bad Helm upgrade
+
+```
+helm history ${RELEASE} -n ${NS}
+helm rollback ${RELEASE} <last-good-revision> -n ${NS} --wait --timeout 15m
+```
+
+`helm rollback` restores the previous chart *and* values, so the site's customisations come back with it.  If the release history itself is unusable, the `/tmp/cassandra-ci-live-values.yaml` from above can be used:
+```
+helm upgrade ${RELEASE} jenkins/jenkins --version ${CHART_VERSION} -n ${NS} \
+    -f /tmp/cassandra-ci-live-values.yaml --wait --timeout 15m
+```
+
+
+### Configuring Local-only Access
 
 If you want only local private access to Jenkins, do the following.
 
-Comment these lines before running `heml upgrade …`
+Comment these lines before running `helm upgrade …`
 ```
 #  serviceType: LoadBalancer
 #  ingress:
 #    enabled: "true"
 ```
-Run the heml upgrade and get the password as usual
+Run the helm upgrade and get the password as usual
 ```
 helm upgrade --install -f values.yaml cassius jenkins/jenkins --wait
 

.jenkins/k8s/jenkins-deployment.yaml

@@ -22,7 +22,11 @@
 persistence:
   enabled: true
   size: "500Gi"
-  # aws needs gp2, gke can be left commented
+  # keep the claim (jobs, credentials, build history) when the release is uninstalled.
+  # deleting it then takes a deliberate `kubectl delete pvc cassius-jenkins`
+  annotations:
+    helm.sh/resource-policy: keep
+  # aws needs gp2, gke can be left commented (add it to your override yaml)
   #storageClass: "gp2"
 controller:
   # To get URL run `kubectl describe svc cassius-jenkins | grep 'LoadBalancer Ingress'`
@@ -31,7 +35,7 @@ controller:
   targetPort: 8080
   ingress:
     enabled: "true"
-    # uncomment and set if you have a "ci-cassandra" dns entry for the jenkins controller
+    # if you have a "ci-cassandra" dns entry for the jenkins controller add the following value to your override yaml
     #hostName: ci-cassandra.<your-domain>
   customJenkinsLabels:
     - controller
@@ -69,13 +73,14 @@ controller:
     - "staticMethod org.codehaus.groovy.runtime.DefaultGroovyMethods inspect java.lang.Object"
     - "staticMethod org.codehaus.groovy.runtime.DefaultGroovyMethods max java.util.Collection"
     - "staticMethod org.codehaus.groovy.runtime.DefaultGroovyMethods putAt java.util.List java.util.List java.lang.Object"
+    - "field hudson.plugins.git.GitSCMBackwardCompatibility branch"
     - "method org.jenkinsci.plugins.workflow.steps.FlowInterruptedException getCauses"
   JCasC:
     configScripts:
       welcome-message: |
         jenkins:
           systemMessage: Welcome to Apache Cassandra
-      # we still need separate jobs because Jenkinsfile can differ (and are read before parameters are applied).
+      # Separate jobs are needed because Jenkinsfiles differ, and are read before parameters are applied.
       #  if a dev branch alters the Jenkinsfile, it will not be picked up by the job – you need to edit the job configuration
       #  see the CAUTION warning in .jenkins/Jenkinsfile
       # TODO: add new version each release branching
@@ -154,6 +159,23 @@ agent:
   node-selector:
     cassandra.jenkins.agent: true
   waitForPodSec: "180"
+  #
+  # Each template below is an opaque string to the helm chart, parsed only by the kubernetes plugin,
+  # so the chart (nor .jenkins/k8s/jenkins-test.sh) can validate it.
+  #
+  # Two traps then to pay attention to:
+  #  - a volume declared under `volumes:` is generated as `volume-N`, and the generated copy wins any
+  #    merge with a raw `yaml:` entry of the same name, so a field set only there is silently dropped.
+  #    Volumes wanting a field the plugin lacks, a sizeLimit for instance, are declared in `yaml:` alone.
+  #  - ephemeral-storage is charged to the pod: every emptyDir, the workspace included, along with the
+  #    containers' writable layers and logs, all against the containers' limits summed.  Declare no
+  #    request and the pod is BestEffort for storage, which the scheduler ignores and the node evicts
+  #    first, reporting only `request is 0`.  Each template below budgets 80Gi (of the ~89Gi that a
+  #    100GiB node allocates) leaving the rest to the node's own image cache and daemonsets.
+  #
+  # After any deploy, validate changes like:
+  #   kubectl get pod -l jenkins/cassius-jenkins-agent -o json | jq '.items[0].spec | {volumes, containers: [.containers[] | {name, resources, volumeMounts}]}'
+  #
   podTemplates:
     agent-dind-small: |
       - name: agent-dind-small
@@ -201,6 +223,9 @@ agent:
             resourceLimitCpu: 2
             resourceRequestMemory: 1G
             resourceLimitMemory: 1G
+            # the workspace emptyDir
+            resourceRequestEphemeralStorage: 10Gi
+            resourceLimitEphemeralStorage: 20Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
           - name: dind
@@ -225,12 +250,13 @@ agent:
             resourceLimitCpu: 4
             resourceRequestMemory: 1G
             resourceLimitMemory: 2400M
+            # docker's images and containers, in the docker-storage emptyDir
+            resourceRequestEphemeralStorage: 40Gi
+            resourceLimitEphemeralStorage: 60Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
         volumes:
-          - emptyDirVolume:
-              memory: 'false'
-              mountPath: /var/lib/docker
+          # /var/lib/docker is not here but in `yaml:` below, the only place it can carry a sizeLimit
           - emptyDirVolume:
               memory: 'false'
               mountPath: /certs
@@ -247,6 +273,18 @@ agent:
                           values:
                             - "true"
                     topologyKey: kubernetes.io/hostname
+            # docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge.
+            # 60Gi bounds the images and their containers alone, of the pod's 80Gi
+            # fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow.
+            volumes:
+              - name: docker-storage
+                emptyDir:
+                  sizeLimit: 60Gi
+            containers:
+              - name: dind
+                volumeMounts:
+                  - name: docker-storage
+                    mountPath: /var/lib/docker
     agent-dind-medium: |
       - name: agent-dind-medium
         label: agent-dind cassandra-medium cassandra-amd64-medium
@@ -291,6 +329,9 @@ agent:
             resourceLimitCpu: 3
             resourceRequestMemory: 1G
             resourceLimitMemory: 2400M
+            # the workspace emptyDir
+            resourceRequestEphemeralStorage: 10Gi
+            resourceLimitEphemeralStorage: 20Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
           - name: dind
@@ -315,12 +356,13 @@ agent:
             resourceLimitCpu: 4
             resourceRequestMemory: 3400M
             resourceLimitMemory: 5G
+            # docker's images and containers, in the docker-storage emptyDir
+            resourceRequestEphemeralStorage: 40Gi
+            resourceLimitEphemeralStorage: 60Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
         volumes:
-          - emptyDirVolume:
-              memory: 'false'
-              mountPath: /var/lib/docker
+          # /var/lib/docker is not here but in `yaml:` below, the only place it can carry a sizeLimit
           - emptyDirVolume:
               memory: 'false'
               mountPath: /certs
@@ -337,6 +379,18 @@ agent:
                           values:
                             - "true"
                     topologyKey: kubernetes.io/hostname
+            # docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge.
+            # 60Gi bounds the images and their containers alone, of the pod's 80Gi
+            # fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow.
+            volumes:
+              - name: docker-storage
+                emptyDir:
+                  sizeLimit: 60Gi
+            containers:
+              - name: dind
+                volumeMounts:
+                  - name: docker-storage
+                    mountPath: /var/lib/docker
     agent-dind-large: |
       - name: agent-dind-large
         label: agent-dind cassandra-large cassandra-amd64-large cassandra-amd64-large-dedicated
@@ -381,6 +435,9 @@ agent:
             resourceLimitCpu: 3
             resourceRequestMemory: 1G
             resourceLimitMemory: 2G
+            # the workspace emptyDir
+            resourceRequestEphemeralStorage: 10Gi
+            resourceLimitEphemeralStorage: 20Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
           - name: dind
@@ -405,12 +462,13 @@ agent:
             resourceLimitCpu: 7
             resourceRequestMemory: 16G
             resourceLimitMemory: 30G
+            # docker's images and containers, in the docker-storage emptyDir
+            resourceRequestEphemeralStorage: 40Gi
+            resourceLimitEphemeralStorage: 60Gi
             ttyEnabled: 'true'
             workingDir: /home/jenkins/agent
         volumes:
-          - emptyDirVolume:
-              memory: 'false'
-              mountPath: /var/lib/docker
+          # /var/lib/docker is not here but in `yaml:` below, the only place it can carry a sizeLimit
           - emptyDirVolume:
               memory: 'false'
               mountPath: /certs
@@ -427,5 +485,17 @@ agent:
                           values:
                             - "true"
                     topologyKey: kubernetes.io/hostname
+            # docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge.
+            # 60Gi bounds the images and their containers alone, of the pod's 80Gi
+            # fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow.
+            volumes:
+              - name: docker-storage
+                emptyDir:
+                  sizeLimit: 60Gi
+            containers:
+              - name: dind
+                volumeMounts:
+                  - name: docker-storage
+                    mountPath: /var/lib/docker
 
 

.jenkins/k8s/jenkins-test.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Validates the CI declarations under .jenkins/ without deploying anything:
+#  - the Jenkinsfile parses as groovy
+#  - jenkins-deployment.yaml renders through the jenkins helm chart
+#  - the yaml embedded in it (agent pod templates, JCasC config scripts) parses
+#
+# Requires: helm, python3 with pyyaml, and either groovy or docker.
+# Run from anywhere: .jenkins/k8s/jenkins-test.sh
+
+set -e
+
+CASSANDRA_DIR="$(cd "$(dirname "$0")/../.." > /dev/null && pwd)"
+JENKINS_DIR="${CASSANDRA_DIR}/.jenkins"
+status=0
+
+command -v helm > /dev/null || { echo "helm must be installed and in the PATH"; exit 1; }
+command -v python3 > /dev/null || { echo "python3 must be installed and in the PATH"; exit 1; }
+python3 -c "import yaml" 2> /dev/null || { echo "python3 pyyaml must be installed: pip install pyyaml"; exit 1; }
+
+echo "== Jenkinsfile groovy syntax"
+# Phases.CONVERSION parses and builds the AST without resolving the pipeline DSL or @NonCPS,
+# neither of which exist outside a jenkins controller
+syntax_check_dir="$(mktemp -d)"
+syntax_check="${syntax_check_dir}/syntax-check.groovy"
+# mktemp gives 0700, which the unprivileged user inside the groovy image cannot traverse
+chmod 755 "${syntax_check_dir}"
+cat > "${syntax_check}" << 'EOF'
+import org.codehaus.groovy.control.CompilationUnit
+import org.codehaus.groovy.control.Phases
+
+def cu = new CompilationUnit()
+args.each { cu.addSource(new File(it)) }
+cu.compile(Phases.CONVERSION)
+println "  ${args.join(', ')} parses"
+EOF
+if command -v groovy > /dev/null ; then
+  groovy "${syntax_check}" "${JENKINS_DIR}/Jenkinsfile" || status=1
+elif command -v docker > /dev/null ; then
+  # absolute paths, the image's working directory is not where the script was mounted
+  docker run --rm -v "${syntax_check_dir}:/check:ro" -v "${JENKINS_DIR}:/jenkins:ro" \
+      groovy:4.0-jdk17 groovy /check/syntax-check.groovy /jenkins/Jenkinsfile || status=1
+else
+  echo "  SKIPPED: neither groovy nor docker found"
+fi
+
+echo "== jenkins-deployment.yaml renders through the helm chart"
+helm repo add jenkins https://charts.jenkins.io > /dev/null
+helm repo update > /dev/null
+# --namespace and a release name only so the chart's templates have something to interpolate
+helm template cassius jenkins/jenkins --namespace default -f "${JENKINS_DIR}/k8s/jenkins-deployment.yaml" > /dev/null \
+  && echo "  jenkins-deployment.yaml renders" || status=1
+
+echo "== yaml embedded in jenkins-deployment.yaml"
+python3 - "${JENKINS_DIR}/k8s" << 'EOF' || status=1
+import sys, yaml
+from pathlib import Path
+
+k8s_dir = Path(sys.argv[1])
+errors = 0
+
+for path in sorted(k8s_dir.glob("*.yaml")):
+    try:
+        values = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    except yaml.YAMLError as error:
+        print(f"  INVALID {path.name}: {error}")
+        errors += 1
+        continue
+    print(f"  {path.name} parses")
+
+    # the values in these two maps are themselves yaml documents, and a chart never validates them —
+    # a misindented pod template reaches the kubernetes plugin and silently loses its agents
+    for keys in (("agent", "podTemplates"), ("controller", "JCasC", "configScripts")):
+        embedded = values
+        for key in keys:
+            embedded = embedded.get(key, {}) if isinstance(embedded, dict) else {}
+        for name, document in (embedded or {}).items():
+            location = f"{path.name} {'.'.join(keys)}.{name}"
+            try:
+                parsed = yaml.safe_load(document)
+            except yaml.YAMLError as error:
+                print(f"  INVALID {location}: {error}")
+                errors += 1
+                continue
+            print(f"  {location} parses")
+            # each pod template may carry a raw kubernetes pod spec in a nested `yaml` key
+            for template in parsed if isinstance(parsed, list) else []:
+                if isinstance(template, dict) and "yaml" in template:
+                    try:
+                        yaml.safe_load(template["yaml"])
+                    except yaml.YAMLError as error:
+                        print(f"  INVALID {location}.yaml: {error}")
+                        errors += 1
+                    else:
+                        print(f"  {location}.yaml parses")
+
+sys.exit(1 if errors else 0)
+EOF
+
+[ 0 -eq ${status} ] && echo "== all .jenkins/ checks passed" || echo "== FAILED"
+exit ${status}

CHANGES.txt

@@ -1,4 +1,5 @@
 5.0.9
+ * Coordinator load-shedding returns OverloadedException without setting streamId, misrouting query responses (CASSANDRA-21508)
  * SAI Component Checksum Validation Should be Segment-Aware (CASSANDRA-21516)
  * Support Python 3.12 and 3.13 in cqlsh (CASSANDRA-20997)
  * Fix AssertionError in hasReplicaWithOngoingRepair when parallel_repair_count > 1 (CASSANDRA-21426)
@@ -11,6 +12,7 @@
 Merged from 4.1:
  * Add Paxos v2 option and informatin in cassandra.yaml (CASSANDRA-21316)
 Merged from 4.0:
+ * Bound declared value length against readable bytes in CBUtil (CASSANDRA-21521)
  * Verify extension type before initializing reflectively-loaded classes (CASSANDRA-21525)
  * Rename conflicting nodetool import --copy-data short option from -p to -cd (CASSANDRA-20214)
  * Fix PasswordObfuscator failing to obfuscate certain passwords (CASSANDRA-21113)

debian/changelog

@@ -2,7 +2,7 @@ cassandra (5.0.9) unstable; urgency=medium
 
   * New release
 
- -- Stefan Miklosovic <smiklosovic@apache.org>  Mon, 13 Jul 2026 14:28:17 +0200
+ -- Stefan Miklosovic <smiklosovic@apache.org>  Tue, 28 Jul 2026 12:57:11 +0200
 
 cassandra (5.0.8) unstable; urgency=medium
 

src/java/org/apache/cassandra/transport/CBUtil.java

@@ -454,6 +454,9 @@ public abstract class CBUtil
         int length = cb.readInt();
         if (length < 0)
             return null;
+        if (length > cb.readableBytes())
+            throw new ProtocolException(String.format("Cannot read value of length %d, only %d bytes remaining in the message",
+                                                      length, cb.readableBytes()));
 
         ByteBuffer buffer = cb.nioBuffer(cb.readerIndex(), length);
         cb.skipBytes(length);
@@ -660,6 +663,9 @@ public abstract class CBUtil
 
     private static byte[] readRawBytes(ByteBuf cb, int length)
     {
+        if (length > cb.readableBytes())
+            throw new ProtocolException(String.format("Cannot read value of length %d, only %d bytes remaining in the message",
+                                                      length, cb.readableBytes()));
         byte[] bytes = new byte[length];
         cb.readBytes(bytes);
         return bytes;

src/java/org/apache/cassandra/transport/CQLMessageHandler.java

@@ -104,7 +104,7 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
 
     interface MessageConsumer<M extends Message>
     {
-        void dispatch(Channel channel, M message, Dispatcher.FlushItemConverter toFlushItem, Overload backpressure);
+        <P> void dispatch(Channel channel, M message, Dispatcher.FlushItemConverter<P> toFlushItem, P param, Overload backpressure);
         boolean hasQueueCapacity();
     }
 
@@ -388,7 +388,7 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
         try
         {
             message = messageDecoder.decode(channel, request);
-            dispatcher.dispatch(channel, message, this::toFlushItem, backpressure);
+            dispatcher.dispatch(channel, message, CQLMessageHandler::toFlushItem, this, backpressure);
             
             // sucessfully delivered a CQL message to the execution
             // stage, so reset the counter of consecutive errors
@@ -484,7 +484,8 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
         // The Dispatcher will call this to obtain the FlushItem to enqueue with its Flusher once
         // a dispatched request has been processed.
 
-        Envelope responseFrame = response.encode(request.getSource().header.version);
+        Envelope.Header header = request.getSource().header;
+        Envelope responseFrame = response.encode(header.version, header.streamId);
         int responseSize = envelopeSize(responseFrame.header);
         ClientMessageSizeMetrics.bytesSent.inc(responseSize);
         ClientMessageSizeMetrics.bytesSentPerResponse.update(responseSize);
@@ -498,9 +499,9 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
 
     private void release(Flusher.FlushItem<Envelope> flushItem)
     {
-        release(flushItem.request.header);
-        flushItem.request.release();
-        flushItem.response.release();
+        release(flushItem.requestEnvelope.header);
+        flushItem.requestEnvelope.release();
+        flushItem.responseEnvelope.release();
     }
 
     private void release(Envelope.Header header)
@@ -522,8 +523,9 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
             if (!extracted.isSuccess())
             {
                 // Hard fail on any decoding error as we can't trust the subsequent frames of
-                // the large message
-                handleError(ProtocolException.toFatalException(extracted.error()));
+                // the large message. The stream id is a best-effort value read before extraction
+                // failed, so route it back where possible rather than defaulting.
+                handleError(ProtocolException.toFatalException(extracted.error()), extracted.streamId());
                 return false;
             }
 
@@ -540,7 +542,7 @@ public class CQLMessageHandler<M extends Message> extends AbstractMessageHandler
                 // not make sense to continue processing subsequent frames
                 handleError(ProtocolException.toFatalException(new OversizedAuthMessageException(
                             MULTI_FRAME_AUTH_ERROR_MESSAGE_PREFIX +
-                            "type = " + header.type + ", size = " + header.bodySizeInBytes)));
+                            "type = " + header.type + ", size = " + header.bodySizeInBytes)), header.streamId);
                 ClientMetrics.instance.markRequestDiscarded();
                 return false;
             }

src/java/org/apache/cassandra/transport/Dispatcher.java

@@ -89,10 +89,9 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
      * The instances of these FlushItem subclasses are specialized to release resources in the
      * right way for the specific pipeline that produced them.
      */
-    // TODO parameterize with FlushItem subclass
-    interface FlushItemConverter
+    public interface FlushItemConverter<P>
     {
-        FlushItem<?> toFlushItem(Channel channel, Message.Request request, Message.Response response);
+        FlushItem<?> toFlushItem(P param, Channel channel, Message.Request request, Message.Response response);
     }
 
     public Dispatcher(boolean useLegacyFlusher)
@@ -101,18 +100,17 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
     }
 
     @Override
-    public void dispatch(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure)
+    public <P> void dispatch(Channel channel, Message.Request request, FlushItemConverter<P> forFlusher, P param, Overload backpressure)
     {
         if (!request.connection().getTracker().isRunning())
         {
             // We can not respond with a custom, transport, or server exceptions since, given current implementation of clients,
             // they will defunct the connection. Without a protocol version bump that introduces an "I am going away message",
             // we have to stick to an existing error code.
-            Message.Response response = ErrorMessage.fromException(new OverloadedException("Server is shutting down"));
-            response.setStreamId(request.getStreamId());
+            Message.Response response = ErrorMessage.fromTransportException(new OverloadedException("Server is shutting down"));
             response.setWarnings(ClientWarn.instance.getWarnings());
             response.attach(request.connection);
-            FlushItem<?> toFlush = forFlusher.toFlushItem(channel, request, response);
+            FlushItem<?> toFlush = forFlusher.toFlushItem(param, channel, request, response);
             flush(toFlush);
             return;
         }
@@ -124,7 +122,7 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
         // Importantly, the authExecutor will handle the AUTHENTICATE message which may be CPU intensive.
         LocalAwareExecutorPlus executor = isAuthQuery ? authExecutor : requestExecutor;
 
-        executor.submit(new RequestProcessor(channel, request, forFlusher, backpressure));
+        executor.submit(new RequestProcessor<>(channel, request, forFlusher, param, backpressure));
         ClientMetrics.instance.markRequestDispatched();
     }
 
@@ -283,20 +281,22 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
      * is the only way we can keep it not wrapped into a callable on SEPExecutor submission path. And we need this
      * functionality for tracking time purposes.
      */
-    public class RequestProcessor implements DebuggableTask.RunnableDebuggableTask
+    public class RequestProcessor<P> implements DebuggableTask.RunnableDebuggableTask
     {
         private final Channel channel;
         private final Message.Request request;
-        private final FlushItemConverter forFlusher;
+        private final FlushItemConverter<P> forFlusher;
+        private final P flusherParam;
         private final Overload backpressure;
 
         private volatile long startTimeNanos;
 
-        public RequestProcessor(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure)
+        public RequestProcessor(Channel channel, Message.Request request, FlushItemConverter<P> forFlusher, P flusherParam, Overload backpressure)
         {
             this.channel = channel;
             this.request = request;
             this.forFlusher = forFlusher;
+            this.flusherParam = flusherParam;
             this.backpressure = backpressure;
         }
 
@@ -304,7 +304,7 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
         public void run()
         {
             startTimeNanos = MonotonicClock.Global.preciseTime.now();
-            processRequest(channel, request, forFlusher, backpressure, new RequestTime(request.createdAtNanos, startTimeNanos));
+            processRequest(channel, request, forFlusher, flusherParam, backpressure, new RequestTime(request.createdAtNanos, startTimeNanos));
         }
 
         @Override
@@ -364,7 +364,7 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
         if (queueTime > DatabaseDescriptor.getNativeTransportTimeout(TimeUnit.NANOSECONDS))
         {
             ClientMetrics.instance.markTimedOutBeforeProcessing();
-            return ErrorMessage.fromException(new OverloadedException("Query timed out before it could start"));
+            return ErrorMessage.fromTransportException(new OverloadedException("Query timed out before it could start"));
         }
 
         if (connection.getVersion().isGreaterOrEqualTo(ProtocolVersion.V4))
@@ -418,8 +418,6 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
         if (request.isTrackable())
             CoordinatorWarnings.done();
 
-        response.setStreamId(request.getStreamId());
-        response.setWarnings(ClientWarn.instance.getWarnings());
         response.attach(connection);
         connection.applyStateTransition(request.type, response.type);
         return response;
@@ -430,9 +428,10 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
      */
     static Message.Response processRequest(Channel channel, Message.Request request, Overload backpressure, RequestTime requestTime)
     {
+        Message.Response response = null;
         try
         {
-            return processRequest((ServerConnection) request.connection(), request, backpressure, requestTime);
+            response = processRequest((ServerConnection) request.connection(), request, backpressure, requestTime);
         }
         catch (Throwable t)
         {
@@ -442,25 +441,25 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
                 CoordinatorWarnings.done();
 
             Predicate<Throwable> handler = ExceptionHandlers.getUnexpectedExceptionHandler(channel, true);
-            ErrorMessage error = ErrorMessage.fromException(t, handler);
-            error.setStreamId(request.getStreamId());
-            error.setWarnings(ClientWarn.instance.getWarnings());
-            return error;
+            response = ErrorMessage.fromExceptionNoStreamId(t, handler);
         }
         finally
         {
+            if (response != null)
+                response.setWarnings(ClientWarn.instance.getWarnings());
             CoordinatorWarnings.reset();
             ClientWarn.instance.resetWarnings();
         }
+        return response;
     }
 
     /**
      * Note: this method is not expected to execute on the netty event loop.
      */
-    void processRequest(Channel channel, Message.Request request, FlushItemConverter forFlusher, Overload backpressure, RequestTime requestTime)
+    <P> void processRequest(Channel channel, Message.Request request, FlushItemConverter<P> forFlusher, P param, Overload backpressure, RequestTime requestTime)
     {
         Message.Response response = processRequest(channel, request, backpressure, requestTime);
-        FlushItem<?> toFlush = forFlusher.toFlushItem(channel, request, response);
+        FlushItem<?> toFlush = forFlusher.toFlushItem(param, channel, request, response);
         Message.logger.trace("Responding: {}, v={}", response, request.connection().getVersion());
         flush(toFlush);
     }
@@ -497,7 +496,7 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
      * for delivering events to registered clients is dependent on protocol version and the configuration
      * of the pipeline. For v5 and newer connections, the event message is encoded into an Envelope,
      * wrapped in a FlushItem and then delivered via the pipeline's flusher, in a similar way to
-     * a Response returned from {@link #processRequest(Channel, Message.Request, FlushItemConverter, Overload, RequestTime)}.
+     * a Response returned from {@link #processRequest(Channel, Message.Request, FlushItemConverter, Object, Overload, RequestTime)}.
      * It's worth noting that events are not generally fired as a direct response to a client request,
      * so this flush item has a null request attribute. The dispatcher itself is created when the
      * pipeline is first configured during protocol negotiation and is attached to the channel for
@@ -511,9 +510,9 @@ public class Dispatcher implements CQLMessageHandler.MessageConsumer<Message.Req
                                            final FrameEncoder.PayloadAllocator allocator)
     {
         return eventMessage -> flush(new FlushItem.Framed(channel,
-                                                          eventMessage.encode(version),
+                                                          eventMessage.encode(version, EventMessage.EVENT_MESSAGE_STREAM_ID), // -1 was set in EventMessage previously
                                                           null,
                                                           allocator,
-                                                          f -> f.response.release()));
+                                                          f -> f.responseEnvelope.release()));
     }
 }

src/java/org/apache/cassandra/transport/Envelope.java

@@ -141,6 +141,11 @@ public class Envelope
         public final Message.Type type;
         public final long bodySizeInBytes;
 
+        public static Header dummy(int streamId, Message.Type type)
+        {
+            return new Header(ProtocolVersion.CURRENT, Flag.deserialize(0), streamId, type, 0);
+        }
+
         private Header(ProtocolVersion version, EnumSet<Flag> flags, int streamId, Message.Type type, long bodySizeInBytes)
         {
             this.version = version;
@@ -242,7 +247,7 @@ public class Envelope
                 // This throws a protocol exception if the version number is unsupported,
                 // the opcode is unknown or invalid flags are set for the version
                 version = ProtocolVersion.decode(versionNum, DatabaseDescriptor.getNativeTransportAllowOlderProtocols());
-                decodedFlags = decodeFlags(version, flags);
+                decodedFlags = decodeFlags(version, flags, streamId);
                 type = Message.Type.fromOpcode(opcode, direction);
                 return new HeaderExtractionResult.Success(new Header(version, decodedFlags, streamId, type, bodyLength));
             }
@@ -256,6 +261,10 @@ public class Envelope
                 // cause the channel to be closed.
                 return new HeaderExtractionResult.Error(e, streamId, bodyLength);
             }
+            catch (ErrorMessage.WrappedException e)
+            {
+                return new HeaderExtractionResult.Error((ProtocolException) e.getCause(), e.getStreamId(), bodyLength);
+            }
         }
 
         public static abstract class HeaderExtractionResult
@@ -354,7 +363,8 @@ public class Envelope
             Message.Direction direction = Message.Direction.extractFromVersion(firstByte);
             int versionNum = firstByte & PROTOCOL_VERSION_MASK;
 
-            ProtocolVersion version;
+            ProtocolVersion version = null;
+            ProtocolException protocolException = null;
             
             try
             {
@@ -362,19 +372,43 @@ public class Envelope
             }
             catch (ProtocolException e)
             {
-                // Skip the remaining useless bytes. Otherwise the channel closing logic may try to decode again. 
-                buffer.skipBytes(readableBytes);
-                throw e;
+                // defer throw to attempt to extract the stream id
+                protocolException = e;
             }
 
             // Wait until we have the complete header
             if (readableBytes < Header.LENGTH)
+            {
+                if (protocolException != null)
+                {
+                    // Skip the remaining useless bytes. Otherwise the channel closing logic may try to decode again.
+                    buffer.skipBytes(readableBytes);
+                    // The header is incomplete, so there is no stream id to recover. Wrap with the unset
+                    // sentinel; the channel-level exception handler sees it has no routable stream id and
+                    // closes the connection rather than emit an unroutable error frame.
+                    throw protocolException;
+                }
                 return null;
+            }
 
             int flags = buffer.getByte(idx++);
-            EnumSet<Header.Flag> decodedFlags = decodeFlags(version, flags);
-
             int streamId = buffer.getShort(idx);
+
+            if (protocolException != null)
+            {
+                // Protocol versions 1 and 2 use a shorter header with a single-byte stream id. Reading a
+                // 16-bit stream id from such a header splices the stream id byte together with the opcode
+                // byte and recovers a bogus id, routing the error to a stream the client never used
+                // (CASSANDRA-21508). A v1/v2 client that downgrades would then never see the error and time
+                // out, so recover the stream id using the attempted version's header layout.
+                int recoveredStreamId = versionNum < ProtocolVersion.V3.asInt() ? buffer.getByte(idx) : streamId;
+                // Skip the remaining useless bytes. Otherwise the channel closing logic may try to decode again.
+                buffer.skipBytes(readableBytes);
+                throw ErrorMessage.wrap(protocolException, recoveredStreamId);
+            }
+
+            EnumSet<Header.Flag> decodedFlags = decodeFlags(version, flags, streamId);
+
             idx += 2;
 
             // This throws a protocol exceptions if the opcode is unknown
@@ -420,13 +454,14 @@ public class Envelope
             return new Envelope(new Header(version, decodedFlags, streamId, type, bodyLength), body);
         }
 
-        private EnumSet<Header.Flag> decodeFlags(ProtocolVersion version, int flags)
+        private EnumSet<Header.Flag> decodeFlags(ProtocolVersion version, int flags, int streamId)
         {
             EnumSet<Header.Flag> decodedFlags = Header.Flag.deserialize(flags);
 
             if (version.isBeta() && !decodedFlags.contains(Header.Flag.USE_BETA))
-                throw new ProtocolException(String.format("Beta version of the protocol used (%s), but USE_BETA flag is unset", version),
-                                            version);
+                throw ErrorMessage.wrap(new ProtocolException(String.format("Beta version of the protocol used (%s), but USE_BETA flag is unset", version),
+                                                              version),
+                                        streamId);
             return decodedFlags;
         }
 

src/java/org/apache/cassandra/transport/ExceptionHandlers.java

@@ -42,6 +42,7 @@ import org.apache.cassandra.exceptions.OversizedCQLMessageException;
 import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.net.FrameEncoder;
 import org.apache.cassandra.transport.messages.ErrorMessage;
+import org.apache.cassandra.transport.messages.ErrorMessage.WithStreamId;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.NoSpamLogger;
 import org.apache.cassandra.utils.Throwables;
@@ -74,23 +75,53 @@ public class ExceptionHandlers
             if (ctx.channel().isOpen())
             {
                 Predicate<Throwable> handler = getUnexpectedExceptionHandler(ctx.channel(), false);
-                ErrorMessage errorMessage = ErrorMessage.fromException(cause, handler);
-                Envelope response = errorMessage.encode(version);
-                FrameEncoder.Payload payload = allocator.allocate(true, CQLMessageHandler.envelopeSize(response.header));
+                // No request in scope at the channel level; a WrappedException cause carries the frame's
+                // stream id and overrides this fallback.
+                WithStreamId withStreamId = ErrorMessage.fromException(cause, handler);
+                ErrorMessage errorMessage = withStreamId.message;
                 try
                 {
-                    response.encodeInto(payload.buffer);
-                    response.release();
-                    payload.finish();
-                    ChannelPromise promise = ctx.newPromise();
-                    // On protocol exception, close the channel as soon as the message has been sent
-                    if (isFatal(cause))
-                        promise.addListener(future -> ctx.close());
-                    ctx.writeAndFlush(payload, promise);
+                    int streamId = withStreamId.streamId;
+                    boolean isFatal = isFatal(cause);
+                    if (streamId == Message.UNSET_STREAM_ID)
+                    {
+                        // No stream id could be recovered, so we have no request to route a response to.
+                        // Close the connection rather than emit an unroutable frame (CASSANDRA-21508).
+                        isFatal = true;
+                        streamId = 0;
+                    }
+
+                    Envelope response = errorMessage.encode(version, streamId);
+                    FrameEncoder.Payload payload = allocator.allocate(true, CQLMessageHandler.envelopeSize(response.header));
+                    try
+                    {
+                        response.encodeInto(payload.buffer);
+                        response.release();
+                        payload.finish();
+                        ChannelPromise promise = ctx.newPromise();
+                        // On a fatal error, close the channel only once the error frame has been written,
+                        // so the client receives the diagnostic before the connection is torn down. Closing
+                        // synchronously here can abort the in-flight flush and drop the frame when the socket
+                        // can't drain it immediately (TCP backpressure, TLS buffering, or a large frame).
+                        // Matches PreV5Handlers.ExceptionHandler and InitialConnectionHandler.
+                        //
+                        // Trade-off of deferring the close (CASSANDRA-21508):
+                        //  - There is a slim chance we send two frames with the same streamId. Responses
+                        //    already queued on this connection will have a chance to flush before the close
+                        //    fires. For the majority of cases, each frame will carry its own unique stream
+                        //    id, so nothing is misrouted. These are valid responses to requests that were
+                        //    correctly-decoded earlier.
+                        if (isFatal)
+                            promise.addListener(future -> ctx.close());
+                        ctx.writeAndFlush(payload, promise);
+                    }
+                    finally
+                    {
+                        payload.release();
+                    }
                 }
                 finally
                 {
-                    payload.release();
                     JVMStabilityInspector.inspectThrowable(cause);
                 }
             }

src/java/org/apache/cassandra/transport/Flusher.java

@@ -35,7 +35,6 @@ import io.netty.channel.EventLoop;
 import org.apache.cassandra.net.FrameEncoder;
 import org.apache.cassandra.net.FrameEncoderCrc;
 import org.apache.cassandra.net.FrameEncoderLZ4;
-import org.apache.cassandra.transport.Message.Response;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.memory.BufferPool;
 
@@ -49,22 +48,22 @@ abstract class Flusher implements Runnable
         Math.min(BufferPool.NORMAL_CHUNK_SIZE,
                  FrameEncoder.Payload.MAX_SIZE - Math.max(FrameEncoderCrc.HEADER_AND_TRAILER_LENGTH, FrameEncoderLZ4.HEADER_AND_TRAILER_LENGTH));
 
-    static class FlushItem<T>
+    public static class FlushItem<T>
     {
         enum Kind {FRAMED, UNFRAMED}
 
         final Kind kind;
         final Channel channel;
-        final T response;
-        final Envelope request;
+        final T responseEnvelope;
+        final Envelope requestEnvelope;
         final Consumer<FlushItem<T>> tidy;
 
-        FlushItem(Kind kind, Channel channel, T response, Envelope request, Consumer<FlushItem<T>> tidy)
+        FlushItem(Kind kind, Channel channel, T responseEnvelope, Envelope requestEnvelope, Consumer<FlushItem<T>> tidy)
         {
             this.kind = kind;
             this.channel = channel;
-            this.request = request;
-            this.response = response;
+            this.requestEnvelope = requestEnvelope;
+            this.responseEnvelope = responseEnvelope;
             this.tidy = tidy;
         }
 
@@ -77,21 +76,21 @@ abstract class Flusher implements Runnable
         {
             final FrameEncoder.PayloadAllocator allocator;
             Framed(Channel channel,
-                   Envelope response,
-                   Envelope request,
+                   Envelope responseEnvelope,
+                   Envelope requestEnvelope,
                    FrameEncoder.PayloadAllocator allocator,
                    Consumer<FlushItem<Envelope>> tidy)
             {
-                super(Kind.FRAMED, channel, response, request, tidy);
+                super(Kind.FRAMED, channel, responseEnvelope, requestEnvelope, tidy);
                 this.allocator = allocator;
             }
         }
 
-        static class Unframed extends FlushItem<Response>
+        static class Unframed extends FlushItem<Envelope>
         {
-            Unframed(Channel channel, Response response, Envelope request, Consumer<FlushItem<Response>> tidy)
+            Unframed(Channel channel, Envelope responseEnvelope, Envelope requestEnvelope, Consumer<FlushItem<Envelope>> tidy)
             {
-                super(Kind.UNFRAMED, channel, response, request, tidy);
+                super(Kind.UNFRAMED, channel, responseEnvelope, requestEnvelope, tidy);
             }
         }
     }
@@ -143,13 +142,13 @@ abstract class Flusher implements Runnable
 
     private void processUnframedResponse(FlushItem.Unframed flush)
     {
-        flush.channel.write(flush.response, flush.channel.voidPromise());
+        flush.channel.write(flush.responseEnvelope, flush.channel.voidPromise());
         channels.add(flush.channel);
     }
 
     private void processFramedResponse(FlushItem.Framed flush)
     {
-        Envelope outbound = flush.response;
+        Envelope outbound = flush.responseEnvelope;
         if (envelopeSize(outbound.header) >= MAX_FRAMED_PAYLOAD_SIZE)
         {
             flushLargeMessage(flush.channel, outbound, flush.allocator);
@@ -157,7 +156,7 @@ abstract class Flusher implements Runnable
         else
         {
             payloads.computeIfAbsent(flush.channel, channel -> new FlushBuffer(channel, flush.allocator, 5))
-                    .add(flush.response);
+                    .add(flush.responseEnvelope);
         }
     }
 

src/java/org/apache/cassandra/transport/InitialConnectionHandler.java

@@ -30,6 +30,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelPromise;
 import io.netty.channel.VoidChannelPromise;
@@ -90,8 +91,7 @@ public class InitialConnectionHandler extends ByteToMessageDecoder
                     supportedOptions.put(StartupMessage.COMPRESSION, compressions);
                     supportedOptions.put(StartupMessage.PROTOCOL_VERSIONS, ProtocolVersion.supportedVersions());
                     SupportedMessage supported = new SupportedMessage(supportedOptions);
-                    supported.setStreamId(inbound.header.streamId);
-                    outbound = supported.encode(inbound.header.version);
+                    outbound = supported.encode(inbound.header.version, inbound.header.streamId);
                     ctx.writeAndFlush(outbound);
                     break;
 
@@ -130,8 +130,8 @@ public class InitialConnectionHandler extends ByteToMessageDecoder
                                 if (null == cause)
                                     cause = new ServerError("Unexpected error establishing connection");
                                 logger.warn("Writing response to STARTUP failed, unable to configure pipeline", cause);
-                                ErrorMessage error = ErrorMessage.fromException(cause);
-                                Envelope response = error.encode(inbound.header.version);
+                                ErrorMessage error = ErrorMessage.fromExceptionNoStreamId(cause);
+                                Envelope response = error.encode(inbound.header.version, inbound.header.streamId);
                                 ChannelPromise closeChannel = AsyncChannelPromise.withListener(ctx, f -> ctx.close());
                                 ctx.writeAndFlush(response, closeChannel);
                                 if (ctx.channel().isOpen())
@@ -151,18 +151,20 @@ public class InitialConnectionHandler extends ByteToMessageDecoder
 
                     final Message.Response response = Dispatcher.processRequest(ctx.channel(), startup, Overload.NONE, Dispatcher.RequestTime.forImmediateExecution());
 
-                    outbound = response.encode(inbound.header.version);
+                    outbound = response.encode(inbound.header.version, inbound.header.streamId);
                     ctx.writeAndFlush(outbound, promise);
                     logger.trace("Configured pipeline: {}", ctx.pipeline());
                     break;
 
                 default:
                     ErrorMessage error =
-                        ErrorMessage.fromException(
+                        ErrorMessage.fromTransportException(
                             new ProtocolException(String.format("Unexpected message %s, expecting STARTUP or OPTIONS",
                                                                 inbound.header.type)));
-                    outbound = error.encode(inbound.header.version);
-                    ctx.writeAndFlush(outbound);
+                    outbound = error.encode(inbound.header.version, inbound.header.streamId);
+                    // An unexpected message during initial connection setup leaves the connection in a
+                    // corrupted state; send the error, then close the connection.
+                    ctx.writeAndFlush(outbound).addListener(ChannelFutureListener.CLOSE);
             }
         }
         finally

src/java/org/apache/cassandra/transport/Message.java

@@ -49,6 +49,12 @@ public abstract class Message
 {
     protected static final Logger logger = LoggerFactory.getLogger(Message.class);
 
+    /**
+     * Sentinel default for a {@link Response}'s stream id;
+     * must be overwritten before {@link #encode}.
+     **/
+    public static final int UNSET_STREAM_ID = Integer.MIN_VALUE;
+
     public interface Codec<M extends Message> extends CBCodec<M> {}
 
     public enum Direction
@@ -144,7 +150,6 @@ public abstract class Message
 
     public final Type type;
     protected Connection connection;
-    private int streamId;
     private Envelope source;
     private Map<String, ByteBuffer> customPayload;
     protected ProtocolVersion forcedProtocolVersion = null;
@@ -164,17 +169,6 @@ public abstract class Message
         return connection;
     }
 
-    public Message setStreamId(int streamId)
-    {
-        this.streamId = streamId;
-        return this;
-    }
-
-    public int getStreamId()
-    {
-        return streamId;
-    }
-
     public void setSource(Envelope source)
     {
         this.source = source;
@@ -198,7 +192,7 @@ public abstract class Message
     @Override
     public String toString()
     {
-        return String.format("(%s:%s:%s)", type, streamId, connection == null ? "null" :  connection.getVersion().asInt());
+        return String.format("(%s:%s)", type, connection == null ? "null" :  connection.getVersion().asInt());
     }
 
     public static abstract class Request extends Message
@@ -326,8 +320,16 @@ public abstract class Message
         }
     }
 
-    public Envelope encode(ProtocolVersion version)
+    public Envelope encode(ProtocolVersion version, int streamId)
     {
+        // A Response's stream id must be stamped before it is serialized to the wire. UNSET_STREAM_ID here
+        // means a server code path produced a response without routing information; sending it would risk
+        // delivering it to an unrelated in-flight request (CASSANDRA-21508). Fail fatally so the connection
+        // is torn down rather than mis-route a response. Checked before the try below so it is not caught and
+        // re-wrapped (which would carry the unset id forward).
+        if (streamId == UNSET_STREAM_ID)
+            throw ProtocolException.toFatalException(new ProtocolException("Attempted to encode a response with an unset stream id: " + this));
+
         EnumSet<Envelope.Header.Flag> flags = EnumSet.noneOf(Envelope.Header.Flag.class);
         @SuppressWarnings("unchecked")
         Codec<Message> codec = (Codec<Message>)this.type.codec;
@@ -414,11 +416,11 @@ public abstract class Message
             if (responseVersion.isBeta())
                 flags.add(Envelope.Header.Flag.USE_BETA);
 
-            return Envelope.create(type, getStreamId(), responseVersion, flags, body);
+            return Envelope.create(type, streamId, responseVersion, flags, body);
         }
         catch (Throwable e)
         {
-            throw ErrorMessage.wrap(e, getStreamId());
+            throw ErrorMessage.wrap(e, streamId);
         }
     }
 
@@ -439,7 +441,6 @@ public abstract class Message
                 throw new ProtocolException("Received frame with CUSTOM_PAYLOAD flag for native protocol version < 4");
 
             Message message = inbound.header.type.codec.decode(inbound.body, inbound.header.version);
-            message.setStreamId(inbound.header.streamId);
             message.setSource(inbound);
             message.setCustomPayload(customPayload);
 

src/java/org/apache/cassandra/transport/PipelineConfigurator.java

@@ -392,7 +392,7 @@ public class PipelineConfigurator
         pipeline.addBefore(INITIAL_HANDLER, MESSAGE_DECOMPRESSOR, Envelope.Decompressor.instance);
         pipeline.addBefore(INITIAL_HANDLER, MESSAGE_COMPRESSOR, Envelope.Compressor.instance);
         pipeline.addBefore(INITIAL_HANDLER, MESSAGE_DECODER, PreV5Handlers.ProtocolDecoder.instance);
-        pipeline.addBefore(INITIAL_HANDLER, MESSAGE_ENCODER, PreV5Handlers.ProtocolEncoder.instance);
+        pipeline.addBefore(INITIAL_HANDLER, MESSAGE_ENCODER, PreV5Handlers.EventMessageEncoder.instance);
         pipeline.addBefore(INITIAL_HANDLER, LEGACY_MESSAGE_PROCESSOR, new PreV5Handlers.LegacyDispatchHandler(dispatcher, queueBackpressure, limits));
         pipeline.remove(INITIAL_HANDLER);
         onNegotiationComplete(pipeline);

src/java/org/apache/cassandra/transport/PreV5Handlers.java

@@ -20,6 +20,7 @@ package org.apache.cassandra.transport;
 
 import java.util.List;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,6 +41,7 @@ import org.apache.cassandra.metrics.ClientMetrics;
 import org.apache.cassandra.net.ResourceLimits;
 import org.apache.cassandra.transport.ClientResourceLimits.Overload;
 import org.apache.cassandra.transport.messages.ErrorMessage;
+import org.apache.cassandra.transport.messages.EventMessage;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 
 import static org.apache.cassandra.transport.CQLMessageHandler.RATE_LIMITER_DELAY_UNIT;
@@ -82,23 +84,26 @@ public class PreV5Handlers
             // The only reason we won't process this message is if checkLimits() throws an OverloadedException.
             // (i.e. Even if backpressure is applied, the current request is allowed to finish.)
             checkLimits(ctx, request);
-            dispatcher.dispatch(ctx.channel(), request, this::toFlushItem, backpressure);
+            dispatcher.dispatch(ctx.channel(), request, this::toFlushItem, ctx, backpressure);
         }
 
         // Acts as a Dispatcher.FlushItemConverter
-        private Flusher.FlushItem.Unframed toFlushItem(Channel channel, Message.Request request, Message.Response response)
+        private Flusher.FlushItem.Unframed toFlushItem(ChannelHandlerContext ctx, Channel channel, Message.Request request, Message.Response response)
         {
-            return new Flusher.FlushItem.Unframed(channel, response, request.getSource(), this::releaseItem);
+            ProtocolVersion version = getConnectionVersion(ctx);
+            Envelope requestEnvelope = request.getSource();
+            Envelope responseEnvelope = response.encode(version, requestEnvelope.header.streamId);
+            return new Flusher.FlushItem.Unframed(channel, responseEnvelope, requestEnvelope, this::releaseItem);
         }
 
-        private void releaseItem(Flusher.FlushItem<Message.Response> item)
+        private void releaseItem(Flusher.FlushItem<Envelope> item)
         {
             // Note: in contrast to the equivalent for V5 protocol, CQLMessageHandler::release(FlushItem item),
             // this does not release the FlushItem's Message.Response. In V4, the buffers for the response's body
             // and serialised header are emitted directly down the Netty pipeline from Envelope.Encoder, so
             // releasing them is handled by the pipeline itself.
-            long itemSize = item.request.header.bodySizeInBytes;
-            item.request.release();
+            long itemSize = item.requestEnvelope.header.bodySizeInBytes;
+            item.requestEnvelope.release();
 
             // since the request has been processed, decrement inflight payload at channel, endpoint and global levels
             channelPayloadBytesInFlight -= itemSize;
@@ -296,14 +301,14 @@ public class PreV5Handlers
      * Simple adaptor to plug CQL message encoding into pre-V5 pipelines
      */
     @ChannelHandler.Sharable
-    public static class ProtocolEncoder extends MessageToMessageEncoder<Message>
+    public static class EventMessageEncoder extends MessageToMessageEncoder<EventMessage>
     {
-        public static final ProtocolEncoder instance = new ProtocolEncoder();
-        private ProtocolEncoder(){}
-        public void encode(ChannelHandlerContext ctx, Message source, List<Object> results)
+        public static final EventMessageEncoder instance = new EventMessageEncoder();
+        private EventMessageEncoder(){}
+        public void encode(ChannelHandlerContext ctx, EventMessage source, List<Object> results)
         {
             ProtocolVersion version = getConnectionVersion(ctx);
-            results.add(source.encode(version));
+            results.add(source.encode(version, EventMessage.EVENT_MESSAGE_STREAM_ID));
         }
     }
 
@@ -326,13 +331,28 @@ public class PreV5Handlers
             if (ctx.channel().isOpen())
             {
                 Predicate<Throwable> handler = ExceptionHandlers.getUnexpectedExceptionHandler(ctx.channel(), false);
-                ErrorMessage errorMessage = ErrorMessage.fromException(cause, handler);
-                ChannelFuture future = ctx.writeAndFlush(errorMessage.encode(getConnectionVersion(ctx)));
-                // On protocol exception, close the channel as soon as the message have been sent.
-                // Most cases of PE are wrapped so the type check below is expected to fail more often than not.
-                // At this moment Fatal exceptions are not thrown in v4, but just as a precaustion we check for them here
-                if (isFatal(cause))
-                    future.addListener((ChannelFutureListener) f -> ctx.close());
+                // No request in scope at the channel level; a WrappedException cause carries the frame's
+                // stream id and overrides this fallback.
+                ErrorMessage.WithStreamId withStreamId = ErrorMessage.fromException(cause, handler);
+
+                if (withStreamId.streamId == Message.UNSET_STREAM_ID)
+                {
+                    // No stream id could be recovered, so we have no request to route a response to.
+                    // Close the connection rather than emit an unroutable frame (CASSANDRA-21508).
+                    ctx.close();
+                }
+                else
+                {
+                    ErrorMessage errorMessage = withStreamId.message;
+                    int streamId = withStreamId.streamId;
+
+                    ChannelFuture future = ctx.writeAndFlush(errorMessage.encode(getConnectionVersion(ctx), streamId));
+                    // On protocol exception, close the channel as soon as the message have been sent.
+                    // Most cases of PE are wrapped so the type check below is expected to fail more often than not.
+                    // At this moment Fatal exceptions are not thrown in v4, but just as a precaustion we check for them here
+                    if (isFatal(cause))
+                        future.addListener((ChannelFutureListener) f -> ctx.close());
+                }
             }
             
             if (DatabaseDescriptor.getClientErrorReportingExclusions().contains(ctx.channel().remoteAddress()))
@@ -354,7 +374,8 @@ public class PreV5Handlers
         }
     }
 
-    private static ProtocolVersion getConnectionVersion(ChannelHandlerContext ctx)
+    @VisibleForTesting
+    static ProtocolVersion getConnectionVersion(ChannelHandlerContext ctx)
     {
         Connection connection = ctx.channel().attr(Connection.attributeKey).get();
         // The only case the connection can be null is when we send the initial STARTUP message

src/java/org/apache/cassandra/transport/SimpleClient.java

@@ -318,7 +318,7 @@ public class SimpleClient implements Closeable
                 for (int i = 0; i < requests.size(); i++)
                 {
                     Message.Request message = requests.get(i);
-                    message.setStreamId(i);
+                    message.setSource(new Envelope(Envelope.Header.dummy(i, message.type), null));
                     message.attach(connection);
                 }
                 lastWriteFuture = channel.writeAndFlush(requests);
@@ -331,7 +331,7 @@ public class SimpleClient implements Closeable
                         throw new RuntimeException("timeout");
                     if (msg instanceof ErrorMessage)
                         throw new RuntimeException((Throwable) ((ErrorMessage) msg).error);
-                    rrMap.put(requests.get(msg.getStreamId()), msg);
+                    rrMap.put(requests.get(msg.getSource().header.streamId), msg);
                 }
             }
             else
@@ -349,6 +349,18 @@ public class SimpleClient implements Closeable
         }
     }
 
+    /**
+     * The stream id to frame an outbound client request with. SimpleClient carries the intended id on the
+     * request's (dummy) source envelope (see {@link #execute(List)} and callers that pipeline requests).
+     * When no source has been assigned, we fall back to 0, which is sufficient for the non-pipelined path
+     * where only a single request is ever in flight.
+     */
+    private static int outboundStreamId(Message message)
+    {
+        Envelope source = message.getSource();
+        return source == null ? 0 : source.header.streamId;
+    }
+
     public interface EventHandler
     {
         void onEvent(Event event);
@@ -404,36 +416,37 @@ public class SimpleClient implements Closeable
             this.largeMessageThreshold = largeMessageThreshold;
         }
 
-        protected void decode(ChannelHandlerContext ctx, Envelope response, List<Object> results)
+        @Override
+        protected void decode(ChannelHandlerContext ctx, Envelope request, List<Object> results)
         {
-            switch(response.header.type)
+            switch(request.header.type)
             {
                 case READY:
                 case AUTHENTICATE:
-                    if (response.header.version.isGreaterOrEqualTo(ProtocolVersion.V5))
+                    if (request.header.version.isGreaterOrEqualTo(ProtocolVersion.V5))
                     {
-                        configureModernPipeline(ctx, response, largeMessageThreshold);
+                        configureModernPipeline(ctx, request, largeMessageThreshold);
                         // consuming the message is done when setting up the pipeline
                     }
                     else
                     {
                         configureLegacyPipeline(ctx);
                         // really just removes self from the pipeline, so pass this message on
-                        ctx.pipeline().context(Envelope.Decoder.class).fireChannelRead(response);
+                        ctx.pipeline().context(Envelope.Decoder.class).fireChannelRead(request);
                     }
                     break;
                 case SUPPORTED:
                     // just pass through
-                    results.add(response);
+                    results.add(request);
                     break;
                 default:
-                    throw new ProtocolException(String.format("Unexpected %s response expecting " +
+                    throw new ProtocolException(String.format("Unexpected %s request expecting " +
                                                               "READY, AUTHENTICATE or SUPPORTED",
-                                                              response.header.type));
+                                                              request.header.type));
             }
         }
 
-        private void configureModernPipeline(ChannelHandlerContext ctx, Envelope response, int largeMessageThreshold)
+        private void configureModernPipeline(ChannelHandlerContext ctx, Envelope request, int largeMessageThreshold)
         {
             logger.info("Configuring modern pipeline");
             ChannelPipeline pipeline = ctx.pipeline();
@@ -455,7 +468,7 @@ public class SimpleClient implements Closeable
 
             CQLMessageHandler.MessageConsumer<Message.Response> responseConsumer = new CQLMessageHandler.MessageConsumer<Message.Response>()
             {
-                public void dispatch(Channel channel, Message.Response message, Dispatcher.FlushItemConverter toFlushItem, Overload backpressure)
+                public <P> void dispatch(Channel channel, Message.Response message, Dispatcher.FlushItemConverter<P> toFlushItem, P param, Overload backpressure)
                 {
                     responseHandler.handleResponse(channel, message);
                 }
@@ -550,15 +563,15 @@ public class SimpleClient implements Closeable
                     ProtocolVersion version = connection == null ? ProtocolVersion.CURRENT : connection.getVersion();
                     SimpleFlusher flusher = new SimpleFlusher(frameEncoder, largeMessageThreshold);
                     for (Message message : (List<Message>) msg)
-                        flusher.enqueue(message.encode(version));
+                        flusher.enqueue(message.encode(version, outboundStreamId(message)));
 
                     flusher.maybeWrite(ctx, promise);
                 }
             });
             pipeline.remove(this);
 
-            Message.Response message = messageDecoder.decode(ctx.channel(), response);
-            responseConsumer.dispatch(channel, message, (ch, req, resp) -> null, Overload.NONE);
+            Message.Response message = messageDecoder.decode(ctx.channel(), request);
+            responseConsumer.dispatch(channel, message, (p, ch, req, resp) -> null, null, Overload.NONE);
         }
 
         private FrameDecoder frameDecoder(ChannelHandlerContext ctx, BufferPoolAllocator allocator)
@@ -603,7 +616,8 @@ public class SimpleClient implements Closeable
             // The only case the connection can be null is when we send the initial STARTUP message (client side thus)
             ProtocolVersion version = connection == null ? ProtocolVersion.CURRENT : connection.getVersion();
             assert messages.size() == 1;
-            results.add(messages.get(0).encode(version));
+            Message message = messages.get(0);
+            results.add(message.encode(version, outboundStreamId(message)));
         }
     }
 
@@ -627,7 +641,7 @@ public class SimpleClient implements Closeable
             pipeline.addLast(HandlerNames.INITIAL_HANDLER, new InitialHandler(version, responseHandler, largeMessageThreshold));
             pipeline.addLast(HandlerNames.MESSAGE_DECODER, PreV5Handlers.ProtocolDecoder.instance);
             pipeline.addLast(HandlerNames.MESSAGE_ENCODER, MessageBatchEncoder.instance);
-            pipeline.addLast(HandlerNames.RESPONSE_HANDLER,  responseHandler);
+            pipeline.addLast(HandlerNames.RESPONSE_HANDLER, responseHandler);
         }
     }
 

src/java/org/apache/cassandra/transport/messages/AuthResponse.java

@@ -93,7 +93,7 @@ public class AuthResponse extends Message.Request
         {
             ClientMetrics.instance.markAuthFailure();
             AuthEvents.instance.notifyAuthFailure(queryState, e);
-            return ErrorMessage.fromException(e);
+            return ErrorMessage.fromTransportException(e);
         }
     }
 }

src/java/org/apache/cassandra/transport/messages/BatchMessage.java

@@ -236,7 +236,7 @@ public class BatchMessage extends Message.Request
         {
             QueryEvents.instance.notifyBatchFailure(prepared, batchType, queryOrIdList, values, options, state, e);
             JVMStabilityInspector.inspectThrowable(e);
-            return ErrorMessage.fromException(e);
+            return ErrorMessage.fromExceptionNoStreamId(e);
         }
     }
 

src/java/org/apache/cassandra/transport/messages/ErrorMessage.java

@@ -394,25 +394,49 @@ public class ErrorMessage extends Message.Response
         this.error = error;
     }
 
-    private ErrorMessage(TransportException error, int streamId)
+    public static ErrorMessage fromTransportException(TransportException e)
     {
-        this(error);
-        setStreamId(streamId);
+        ErrorMessage message = new ErrorMessage(e);
+        if (e instanceof ProtocolException)
+        {
+            // if the driver attempted to connect with a protocol version not supported then
+            // respond with the appropiate version, see ProtocolVersion.decode()
+            ProtocolVersion forcedProtocolVersion = ((ProtocolException) e).getForcedProtocolVersion();
+            if (forcedProtocolVersion != null)
+                message.forcedProtocolVersion = forcedProtocolVersion;
+        }
+        return message;
     }
 
-    public static ErrorMessage fromException(Throwable e)
+    public static ErrorMessage fromExceptionNoStreamId(Throwable e)
     {
-        return fromException(e, null);
+        return fromExceptionNoStreamId(e, null);
     }
 
+    public static ErrorMessage fromExceptionNoStreamId(Throwable e, Predicate<Throwable> unexpectedExceptionHandler)
+    {
+        return fromException(e, unexpectedExceptionHandler).message;
+    }
+
+    public static class WithStreamId
+    {
+        public final ErrorMessage message;
+        public final int streamId;
+
+        WithStreamId(ErrorMessage message, int streamId)
+        {
+            this.message = message;
+            this.streamId = streamId;
+        }
+    }
     /**
      * @param e the exception
      * @param unexpectedExceptionHandler a callback for handling unexpected exceptions. If null, or if this
      *                                   returns false, the error is logged at ERROR level via sl4fj
      */
-    public static ErrorMessage fromException(Throwable e, Predicate<Throwable> unexpectedExceptionHandler)
+    public static WithStreamId fromException(Throwable e, Predicate<Throwable> unexpectedExceptionHandler)
     {
-        int streamId = 0;
+        int streamId = UNSET_STREAM_ID;
 
         // Netty will wrap exceptions during decoding in a CodecException. If the cause was one of our ProtocolExceptions
         // or some other internal exception, extract that and use it.
@@ -440,23 +464,15 @@ public class ErrorMessage extends Message.Response
 
         if (e instanceof TransportException)
         {
-            ErrorMessage message = new ErrorMessage((TransportException) e, streamId);
-            if (e instanceof ProtocolException)
-            {
-                // if the driver attempted to connect with a protocol version not supported then
-                // respond with the appropiate version, see ProtocolVersion.decode()
-                ProtocolVersion forcedProtocolVersion = ((ProtocolException) e).getForcedProtocolVersion();
-                if (forcedProtocolVersion != null)
-                    message.forcedProtocolVersion = forcedProtocolVersion;
-            }
-            return message;
+            ErrorMessage message = fromTransportException((TransportException) e);
+            return new WithStreamId(message, streamId);
         }
 
         // Unexpected exception
         if (unexpectedExceptionHandler == null || !unexpectedExceptionHandler.apply(e))
             logger.error("Unexpected exception during request", e);
 
-        return new ErrorMessage(new ServerError(e), streamId);
+        return new WithStreamId(new ErrorMessage(new ServerError(e)), streamId);
     }
 
     @Override

src/java/org/apache/cassandra/transport/messages/EventMessage.java

@@ -25,6 +25,8 @@ import org.apache.cassandra.transport.ProtocolVersion;
 
 public class EventMessage extends Message.Response
 {
+    public static final int EVENT_MESSAGE_STREAM_ID = -1;
+
     public static final Message.Codec<EventMessage> codec = new Message.Codec<EventMessage>()
     {
         public EventMessage decode(ByteBuf body, ProtocolVersion version)
@@ -49,7 +51,6 @@ public class EventMessage extends Message.Response
     {
         super(Message.Type.EVENT);
         this.event = event;
-        this.setStreamId(-1);
     }
 
     @Override

src/java/org/apache/cassandra/transport/messages/ExecuteMessage.java

@@ -206,7 +206,7 @@ public class ExecuteMessage extends Message.Request
         {
             QueryEvents.instance.notifyExecuteFailure(prepared, options, state, e);
             JVMStabilityInspector.inspectThrowable(e);
-            return ErrorMessage.fromException(e);
+            return ErrorMessage.fromExceptionNoStreamId(e);
         }
     }
 

src/java/org/apache/cassandra/transport/messages/PrepareMessage.java

@@ -50,12 +50,13 @@ public class PrepareMessage extends Message.Request
         {
             String query = CBUtil.readLongString(body);
             String keyspace = null;
-            if (version.isGreaterOrEqualTo(ProtocolVersion.V5)) {
+            if (version.isGreaterOrEqualTo(ProtocolVersion.V5))
+            {
                 // If flags grows, we may want to consider creating a PrepareOptions class with an internal codec
                 // class that handles flags and options of the prepare message. Since there's only one right now,
                 // we just take care of business here.
 
-                int flags = (int)body.readUnsignedInt();
+                int flags = (int) body.readUnsignedInt();
                 if ((flags & 0x1) == 0x1)
                 {
                     keyspace = CBUtil.readString(body);
@@ -73,8 +74,11 @@ public class PrepareMessage extends Message.Request
             {
                 // If we have no keyspace, write out a 0-valued flag field.
                 if (msg.keyspace == null)
+                {
                     dest.writeInt(0x0);
-                else {
+                }
+                else
+                {
                     dest.writeInt(0x1);
                     CBUtil.writeAsciiString(msg.keyspace, dest);
                 }
@@ -133,7 +137,7 @@ public class PrepareMessage extends Message.Request
         {
             QueryEvents.instance.notifyPrepareFailure(null, query, state, e);
             JVMStabilityInspector.inspectThrowable(e);
-            return ErrorMessage.fromException(e);
+            return ErrorMessage.fromExceptionNoStreamId(e);
         }
     }
 

src/java/org/apache/cassandra/transport/messages/QueryMessage.java

@@ -128,7 +128,7 @@ public class QueryMessage extends Message.Request
             JVMStabilityInspector.inspectThrowable(e);
             if (!((e instanceof RequestValidationException) || (e instanceof RequestExecutionException)))
                 logger.error("Unexpected error during query", e);
-            return ErrorMessage.fromException(e);
+            return ErrorMessage.fromExceptionNoStreamId(e);
         }
     }
 

test/burn/org/apache/cassandra/transport/DriverBurnTest.java

@@ -327,10 +327,10 @@ public class DriverBurnTest extends CQLTester
         SimpleStatement request = generateQueryStatement(0, requestCaps);
         ResultMessage.Rows response = generateRows(0, responseCaps);
         QueryMessage requestMessage = generateQueryMessage(0, requestCaps, version);
-        Envelope message = requestMessage.encode(version);
+        Envelope message = requestMessage.encode(version, 0);
         int requestSize = message.body.readableBytes();
         message.release();
-        message = response.encode(version);
+        message = response.encode(version, 0);
         int responseSize = message.body.readableBytes();
         message.release();
         Message.Type.QUERY.unsafeSetCodec(new Message.Codec<QueryMessage>() {

test/burn/org/apache/cassandra/transport/SimpleClientPerfTest.java

@@ -148,10 +148,10 @@ public class SimpleClientPerfTest
     {
         ResultMessage.Rows response = generateRows(0, responseCaps);
         QueryMessage requestMessage = generateQueryMessage(0, requestCaps, version);
-        Envelope message = requestMessage.encode(version);
+        Envelope message = requestMessage.encode(version, 0);
         int requestSize = message.body.readableBytes();
         message.release();
-        message = response.encode(version);
+        message = response.encode(version, 0);
         int responseSize = message.body.readableBytes();
         message.release();
 

test/distributed/org/apache/cassandra/distributed/test/StreamIdMisrouteTest.java

@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.implementation.MethodDelegation;
+import net.bytebuddy.implementation.bind.annotation.SuperCall;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.statements.SelectStatement;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.exceptions.OverloadedException;
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.Dispatcher;
+import org.apache.cassandra.transport.Envelope;
+import org.apache.cassandra.transport.Message;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.transport.SimpleClient;
+import org.apache.cassandra.transport.messages.ErrorMessage;
+import org.apache.cassandra.transport.messages.QueryMessage;
+import org.apache.cassandra.transport.messages.ResultMessage;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+import static org.apache.cassandra.config.DatabaseDescriptor.clientInitialization;
+import static org.apache.cassandra.config.DatabaseDescriptor.getNativeTransportPort;
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+
+/**
+ * Regression test for CASSANDRA-21508. A coordinator that load-sheds a request which waited past its
+ * native-transport queue deadline must stamp the OVERLOADED error with the timed-out request's own
+ * stream id. Before the fix the error went out on stream id 0, which misroutes it to an unrelated
+ * in-flight request and can escalate into the client-side "column-shift" read corruption (a value
+ * from one table decoded against another query's column definitions under the v5 skip-metadata
+ * optimization).
+ *
+ * <p>The original defect: {@code Dispatcher.processRequest} load-sheds a request that has waited in the
+ * Native-Transport-Requests queue longer than {@code native_transport_timeout} by returning
+ *
+ * <pre>
+ *   ErrorMessage.fromException(new OverloadedException("Query timed out before it could start"))
+ * </pre>
+ * <p>
+ * without calling {@code setStreamId(request.getStreamId())}. {@code ErrorMessage.streamId} therefore
+ * kept its default of 0, and the error frame was written to the client on stream id 0 instead of the
+ * timed-out request's real stream id. The fix routes every response through a central stamping step
+ * and makes {@code ErrorMessage.fromException} require the stream id at the call site.
+ *
+ * <p>{@link #loadShedErrorIsStampedWithRequestStreamId()} proves the fix on the wire: a request sent on
+ * a non-zero stream id is load-shed and the OVERLOADED error comes back on that same stream id (not 0).
+ * Against the unpatched server this assertion fails with the error arriving on stream id 0.
+ *
+ * <p>Why the stream id matters: on a busy connection stream id 0 is almost always in use, so an error
+ * mis-stamped with 0 is applied to an unrelated in-flight request. The client then frees and reuses
+ * stream id 0 for a new query, and when the original query's real rows arrive on it they are decoded
+ * positionally against the reusing query's cached column definitions (skip-metadata carries no column
+ * info) - shifting each value into the wrong column and either throwing in a codec or silently
+ * returning a plausible-but-wrong value.
+ */
+public class StreamIdMisrouteTest extends TestBaseImpl
+{
+    private static final int TIMED_OUT_STREAM_ID = 42; // any non-zero id; before the fix the reply was forced to 0
+
+    @BeforeClass
+    public static void initClientSide()
+    {
+        // SimpleClient runs in the test classloader; it needs DatabaseDescriptor initialized here to
+        // build the native-protocol pipeline. Without this, connect() fails with a ClosedChannelException.
+        clientInitialization();
+    }
+
+    /**
+     * Proves the fix directly: under native-transport queue backlog, the load-shed OVERLOADED error is
+     * returned on the stream id of the request that actually timed out, not on stream id 0. Against the
+     * unpatched server this fails with the error arriving on stream id 0.
+     */
+    @Test
+    public void loadShedErrorIsStampedWithRequestStreamId() throws Throwable
+    {
+        try (Cluster cluster = init(Cluster.build().withNodes(1)
+                                           .withInstanceInitializer(SlowSelect::install)
+                                           .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL)
+                                                                       // one NTR thread so a slow query blocks the queue head
+                                                                       .set("native_transport_max_threads", 1)
+                                                                       // short deadline so a queued request is shed quickly
+                                                                       .set("native_transport_timeout", "150ms")
+                                                                       .set("read_request_timeout", "500ms")
+                                                                       .set("range_request_timeout", "500ms"))
+                                           .start()))
+        {
+            cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int PRIMARY KEY, v int)"));
+
+            InetSocketHost host = hostAndPort(cluster);
+            ExecutorService executor = Executors.newCachedThreadPool();
+
+            // Two independent connections. A SimpleClient's response queue is a SynchronousQueue, so
+            // concurrent execute() calls on ONE client would race; we use one client to build the
+            // backlog and a second, dedicated client for the request whose stream id we assert on.
+            try (SimpleClient filler = SimpleClient.builder(host.address, host.port)
+                                                   .protocolVersion(ProtocolVersion.V5).build().connect(false);
+                 SimpleClient victim = SimpleClient.builder(host.address, host.port)
+                                                   .protocolVersion(ProtocolVersion.V5).build().connect(false))
+            {
+                // Warm up both connections while the server is still fast.
+                filler.execute(query(withKeyspace("SELECT * FROM %s.tbl"), 1));
+                victim.execute(query(withKeyspace("SELECT * FROM %s.tbl"), 1));
+
+                // From here every non-internal SELECT sleeps 1s, far past the 150ms queue deadline.
+                cluster.get(1).runOnInstance(() -> Assert.assertTrue(SlowSelect.enabled.compareAndSet(false, true)));
+
+                // Saturate the single NTR worker and pile several requests behind it, so anything
+                // enqueued now waits well beyond native_transport_timeout before a worker frees up.
+                List<Future<?>> backlog = new ArrayList<>();
+                for (int i = 0; i < 8; i++)
+                {
+                    int streamId = 10 + i;
+                    backlog.add(executor.submit(() ->
+                                                filler.execute(query(withKeyspace("SELECT * FROM %s.tbl"), streamId), false)));
+                }
+
+                // Let the backlog form and the queue-time clock run past the 150ms deadline.
+                TimeUnit.MILLISECONDS.sleep(800);
+
+                // This request enqueues behind a queue that is already older than the deadline, so the
+                // worker load-sheds it immediately when it is dequeued. It goes out on stream id 42.
+                Message.Response shed =
+                victim.execute(query(withKeyspace("SELECT * FROM %s.tbl"), TIMED_OUT_STREAM_ID), false);
+
+                Assert.assertTrue("Expected an OVERLOADED error, got: " + shed,
+                                  shed instanceof ErrorMessage
+                                  && ((ErrorMessage) shed).error instanceof OverloadedException);
+
+                // The fix: the request went out on stream id 42, and the load-shed error now comes back
+                // on stream id 42 too because Dispatcher stamps every response with the request's id.
+                Assert.assertEquals("Load-shed OVERLOADED error should carry the request's own stream id (" +
+                                    TIMED_OUT_STREAM_ID + ')',
+                                    TIMED_OUT_STREAM_ID, shed.getSource().header.streamId);
+
+                // Drain the backlog so the connection can close cleanly.
+                cluster.get(1).runOnInstance(() -> SlowSelect.enabled.set(false));
+                for (Future<?> f : backlog)
+                {
+                    try
+                    {
+                        f.get(30, TimeUnit.SECONDS);
+                    }
+                    catch (Exception ignored)
+                    { /* shed/timed out */ }
+                }
+            }
+            finally
+            {
+                cluster.get(1).runOnInstance(() -> SlowSelect.enabled.set(false));
+                executor.shutdownNow();
+            }
+        }
+    }
+
+    // ------------------------------------------------------------------------------------------------
+    // Helpers
+    // ------------------------------------------------------------------------------------------------
+
+    private static QueryMessage query(String cql, int streamId)
+    {
+        QueryMessage msg = new QueryMessage(cql, QueryOptions.forInternalCalls(
+        org.apache.cassandra.db.ConsistencyLevel.ONE, Collections.emptyList()));
+        msg.setSource(new Envelope(Envelope.Header.dummy(streamId, Message.Type.QUERY), null));
+        return msg;
+    }
+
+    private static InetSocketHost hostAndPort(Cluster cluster)
+    {
+        // Node 1's native transport binds on its broadcast address; the port is whatever the in-JVM
+        // provisioning assigned (9042 for the default multi-interface strategy). Derive both from the
+        // instance config so this works regardless of provisioning strategy.
+        String address = cluster.get(1).config().broadcastAddress().getAddress().getHostAddress();
+        int port = cluster.get(1).callOnInstance(
+        () -> getNativeTransportPort());
+        return new InetSocketHost(address, port);
+    }
+
+    private static final class InetSocketHost
+    {
+        final String address;
+        final int port;
+
+        InetSocketHost(String address, int port)
+        {
+            this.address = address;
+            this.port = port;
+        }
+    }
+
+    /**
+     * ByteBuddy interceptor that makes client-issued SELECTs sleep past the queue deadline, so a
+     * request queued behind one is load-shed by Dispatcher. Mirrors OverloadTest.SlowSelect.
+     */
+    public static class SlowSelect
+    {
+        static final AtomicBoolean enabled = new AtomicBoolean(false);
+
+        static void install(ClassLoader cl, int nodeNumber)
+        {
+            new ByteBuddy().rebase(SelectStatement.class)
+                           .method(named("execute").and(takesArguments(QueryState.class, QueryOptions.class, Dispatcher.RequestTime.class)))
+                           .intercept(MethodDelegation.to(SlowSelect.class))
+                           .make()
+                           .load(cl, ClassLoadingStrategy.Default.INJECTION);
+        }
+
+        @SuppressWarnings("unused")
+        public static ResultMessage.Rows execute(QueryState state, QueryOptions options,
+                                                 Dispatcher.RequestTime requestTime,
+                                                 @SuperCall Callable<ResultMessage.Rows> zuper) throws Exception
+        {
+            if (enabled.get() && !state.getClientState().isInternal)
+                TimeUnit.SECONDS.sleep(1);
+            return zuper.call();
+        }
+    }
+}

test/distributed/org/apache/cassandra/distributed/test/UnableToParseClientMessageTest.java

@@ -184,9 +184,9 @@ public class UnableToParseClientMessageTest extends TestBaseImpl
         }
 
         @Override
-        public Envelope encode(ProtocolVersion version)
+        public Envelope encode(ProtocolVersion version, int streamId)
         {
-            Envelope base = super.encode(version);
+            Envelope base = super.encode(version, streamId);
             return new CustomHeaderEnvelope(base.header, base.body, headerEncoded);
         }
     }
@@ -227,7 +227,7 @@ public class UnableToParseClientMessageTest extends TestBaseImpl
         }
 
         @Override
-        public Envelope encode(ProtocolVersion version)
+        public Envelope encode(ProtocolVersion version, int streamId)
         {
             Codec<?> originalCodec = type.codec;
             try
@@ -254,7 +254,7 @@ public class UnableToParseClientMessageTest extends TestBaseImpl
                     }
                 });
 
-                return super.encode(version);
+                return super.encode(version, streamId);
             }
             finally
             {

test/unit/org/apache/cassandra/transport/CBUtilReadValueNativeProtocolTest.java

@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.transport;
+
+import java.util.Objects;
+
+import org.assertj.core.api.Assertions;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.service.QueryState;
+import org.apache.cassandra.transport.messages.ErrorMessage;
+import org.apache.cassandra.transport.messages.OptionsMessage;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
+/**
+ * If a client sends an AUTH_RESPONSE whose declared SASL-token length is {@link Integer#MAX_VALUE},
+ * {@link CBUtil#readValue(ByteBuf)} must reject it against the readable byte count rather than
+ * attempting the allocation.
+ * <p>
+ * This reproduces a pre-authentication unbounded-allocation DoS: on unpatched code the read
+ * allocates {@code new byte[Integer.MAX_VALUE]} and the JVM throws OutOfMemoryError; on fixed
+ * code the client receives an ERROR containing "Cannot read value of length 2147483647" and the
+ * server keeps running. The V4 (pre-V5) decoder path is the one the original report exercised.
+ */
+public class CBUtilReadValueNativeProtocolTest extends NativeProtocolLimitsTestBase
+{
+    public CBUtilReadValueNativeProtocolTest()
+    {
+        // V4 / pre-V5 path is the one the report and the reproducer netcat PoC exercise.
+        super(ProtocolVersion.V4);
+    }
+
+    @BeforeClass
+    public static void setUp()
+    {
+        requireNetwork();
+    }
+
+    @Test
+    public void authResponseWithUnboundedTokenLengthIsRejected()
+    {
+        ByteBuf maliciousBody = Unpooled.buffer(4);
+        maliciousBody.writeInt(Integer.MAX_VALUE); // 0x7f ff ff ff
+
+        try (SimpleClient client = client())
+        {
+            Message.Response response = client.execute(new CustomBodyMessage(Message.Type.AUTH_RESPONSE,
+                                                                             maliciousBody),
+                                                       false);
+
+            Assertions.assertThat(response.type)
+                      .as("Server must respond with an ERROR for the malformed AUTH_RESPONSE")
+                      .isEqualTo(Message.Type.ERROR);
+            Assertions.assertThat(((ErrorMessage) response).error.getMessage())
+                      .as("ERROR payload must reference the CBUtil bound check, not OutOfMemoryError")
+                      .contains("Cannot read value of length 2147483647");
+
+            // Sanity-check the server is still alive after the malicious frame: a fresh
+            // OPTIONS roundtrip on a new connection should succeed.
+            try (SimpleClient probe = client())
+            {
+                Message.Response options = probe.execute(new OptionsMessage(), true);
+                Assertions.assertThat(options.type).isEqualTo(Message.Type.SUPPORTED);
+            }
+        }
+    }
+
+    /**
+     * Replaces AUTH_RESPONSE's encoder with one that writes a caller-supplied raw body, so
+     * SimpleClient transmits exactly the bytes we want for the test. The server still uses
+     * the original AuthResponse.Codec.decode on receive, which is what we want — that's the
+     * code path under test.
+     */
+    private static final class CustomBodyMessage extends Message.Request
+    {
+        private final ByteBuf body;
+
+        CustomBodyMessage(Message.Type type, ByteBuf body)
+        {
+            super(type);
+            this.body = Objects.requireNonNull(body);
+        }
+
+        @Override
+        public Envelope encode(ProtocolVersion version, int streamId)
+        {
+            Message.Codec<?> originalCodec = type.codec;
+            try
+            {
+                setCodec(type, new Message.Codec<>()
+                {
+                    @Override
+                    public Message decode(ByteBuf in, ProtocolVersion v)
+                    {
+                        return originalCodec.decode(in, v);
+                    }
+
+                    @Override
+                    public void encode(Message message, ByteBuf dest, ProtocolVersion v)
+                    {
+                        dest.writeBytes(body, body.readerIndex(), body.readableBytes());
+                    }
+
+                    @Override
+                    public int encodedSize(Message message, ProtocolVersion v)
+                    {
+                        return body.readableBytes();
+                    }
+                });
+                return super.encode(version, streamId);
+            }
+            finally
+            {
+                setCodec(type, originalCodec);
+            }
+        }
+
+        @Override
+        protected Message.Response execute(QueryState queryState,
+                                           Dispatcher.RequestTime requestTime,
+                                           boolean traceRequest)
+        {
+            throw new AssertionError("execute not expected for a malformed AUTH_RESPONSE");
+        }
+    }
+
+    private static void setCodec(Message.Type type, Message.Codec<?> codec)
+    {
+        try
+        {
+            type.unsafeSetCodec(codec);
+        }
+        catch (NoSuchFieldException | IllegalAccessException e)
+        {
+            throw new AssertionError(e);
+        }
+    }
+}

test/unit/org/apache/cassandra/transport/CQLConnectionTest.java

@@ -20,7 +20,9 @@ package org.apache.cassandra.transport;
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.net.InetSocketAddress;
 import java.net.ServerSocket;
+import java.net.SocketAddress;
 import java.nio.ByteBuffer;
 import java.security.SecureRandom;
 import java.util.*;
@@ -40,8 +42,10 @@ import io.netty.bootstrap.Bootstrap;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import io.netty.channel.*;
+import io.netty.channel.embedded.EmbeddedChannel;
 import io.netty.channel.nio.NioEventLoopGroup;
 import io.netty.handler.codec.MessageToMessageDecoder;
+import io.netty.handler.codec.MessageToMessageEncoder;
 import org.apache.cassandra.auth.AllowAllAuthenticator;
 import org.apache.cassandra.auth.AllowAllAuthorizer;
 import org.apache.cassandra.auth.AllowAllNetworkAuthorizer;
@@ -64,6 +68,7 @@ import static org.apache.cassandra.config.EncryptionOptions.TlsEncryptionPolicy.
 import static org.apache.cassandra.io.util.FileUtils.ONE_MIB;
 import static org.apache.cassandra.net.FramingTest.randomishBytes;
 import static org.apache.cassandra.transport.Flusher.MAX_FRAMED_PAYLOAD_SIZE;
+import static org.apache.cassandra.transport.PreV5Handlers.getConnectionVersion;
 import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeCondition;
 import static org.apache.cassandra.utils.concurrent.NonBlockingRateLimiter.NO_OP_LIMITER;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -147,6 +152,45 @@ public class CQLConnectionTest
         observer.verifier().accept(0);
     }
 
+    @Test
+    public void handleIncompleteHeaderErrorDuringNegotiation() throws Throwable
+    {
+        // CASSANDRA-21508: a partial header (fewer than Header.LENGTH=9 bytes) carrying an unsupported
+        // protocol version gives the server no stream id to route an error back to. Rather than emit an
+        // unroutable error frame (which could be misapplied to another request), the server closes the
+        // connection. Here the client observes a closed connection with no error frame.
+        int messageCount = 0;
+        Codec codec = Codec.crc(alloc);
+        AllocationObserver observer = new AllocationObserver();
+        InboundProxyHandler.Controller controller = new InboundProxyHandler.Controller();
+        // Truncate the client's STARTUP to a partial header (< 9 bytes) and set an unsupported version.
+        controller.withPayloadTransform(msg -> {
+            ByteBuf bb = (ByteBuf) msg;
+            ByteBuf truncated = bb.copy(0, 4);
+            truncated.setByte(0, 99 & Envelope.PROTOCOL_VERSION_MASK);
+            bb.release();
+            return truncated;
+        });
+
+        ServerConfigurator configurator = ServerConfigurator.builder()
+                                                            .withAllocationObserver(observer)
+                                                            .withProxyController(controller)
+                                                            .build();
+        Server server = server(configurator);
+        Client client = new Client(codec, messageCount);
+        server.start();
+        client.connect(address, port);
+
+        // The server cannot recover a stream id from the incomplete header, so it closes the connection
+        // without sending an error frame.
+        assertFalse(client.isConnected());
+        assertThat(client.getConnectionError()).isNull();
+        server.stop();
+
+        // the failure happens before any capacity is allocated
+        observer.verifier().accept(0);
+    }
+
     @Test
     public void handleFrameCorruptionAfterNegotiation() throws Throwable
     {
@@ -218,6 +262,43 @@ public class CQLConnectionTest
         testFrameCorruption(1, Codec.crc(alloc), envelopeProvider, corruptor, totalBytesPerEnvelope, errorCheck);
     }
 
+    @Test
+    public void fatalErrorFrameIsFlushedBeforeChannelClose()
+    {
+        // CASSANDRA-21508: the post-V5 exception handler must close the connection only AFTER the diagnostic
+        // error frame has been flushed. If it closes synchronously right after writeAndFlush, a write that
+        // can't drain immediately (TCP backpressure, TLS, large frame) is aborted and the client never sees
+        // the error. Simulate "write cannot complete synchronously" with an outbound handler that captures
+        // the write promise without completing it - the handler must not have closed the channel yet.
+        StallingOutboundHandler stall = new StallingOutboundHandler();
+        EmbeddedChannel channel = new EmbeddedChannel()
+        {
+            // client_error_reporting_exclusions (SubnetGroups.contains) requires a real InetSocketAddress;
+            // EmbeddedChannel's default remote address is not one, so supply a loopback address.
+            @Override
+            protected SocketAddress remoteAddress0()
+            {
+                return new InetSocketAddress("127.0.0.1", 9042);
+            }
+        };
+        channel.pipeline().addLast(stall);
+        channel.pipeline().addLast(ExceptionHandlers.postV5Handler(FrameEncoderCrc.instance.allocator(),
+                                                                   ProtocolVersion.V5));
+
+        // Fatal protocol error, no recoverable stream id -> handler encodes an error frame and fatally closes.
+        channel.pipeline().fireExceptionCaught(ProtocolException.toFatalException(new ProtocolException("boom")));
+
+        // The error frame was handed to the outbound pipeline...
+        assertNotNull("expected the handler to write an error frame", stall.writePromise);
+        // ...but since that write has not completed, the channel must still be open. The buggy synchronous
+        // ctx.close() would have already closed it here.
+        assertTrue("channel must not close before the error frame is flushed", channel.isOpen());
+
+        // Once the write completes, the deferred close fires.
+        stall.writePromise.setSuccess();
+        assertFalse("channel should close after the error frame is flushed", channel.isOpen());
+    }
+
     @Test
     public void testAquireAndRelease()
     {
@@ -293,6 +374,48 @@ public class CQLConnectionTest
         runTest(configurator, codec, messageCount, envelopeProvider, responseMatcher, observer.verifier());
     }
 
+    @Test
+    public void testRecoverableBetaFlagEnvelopeErrors()
+    {
+        // CASSANDRA-21508: a V6 (beta) envelope header with the USE_BETA flag unset is an
+        // invalid but *recoverable* protocol error, exactly like an unknown opcode. The server should
+        // return an ERROR on the offending stream id and continue processing subsequent envelopes on the
+        // same connection.
+
+        // every other message advertises V6 with USE_BETA unset and should error while extracting the header
+        IntPredicate shouldError = i -> i % 2 == 0;
+        testBetaFlagEnvelopeErrors(10, shouldError, Codec.crc(alloc));
+        testBetaFlagEnvelopeErrors(10, shouldError, Codec.lz4(alloc));
+
+        testBetaFlagEnvelopeErrors(100, shouldError, Codec.crc(alloc));
+    }
+
+    private void testBetaFlagEnvelopeErrors(int messageCount, IntPredicate shouldError, Codec codec)
+    {
+        TestConsumer consumer = new TestConsumer(new ResultMessage.Void(), codec.encoder);
+        AllocationObserver observer = new AllocationObserver(false);
+        Message.Decoder<Message.Request> decoder = new FixedDecoder();
+
+        // Mutate the erroring streams' headers to advertise the V6 (beta) version with the USE_BETA flag
+        // cleared. In the envelope header, byte 0 is the (direction & version) byte and byte 1 is the flags.
+        int betaBit = 1 << Envelope.Header.Flag.USE_BETA.ordinal();
+        IntFunction<Envelope> envelopeProvider = mutatedEnvelopeProvider(shouldError, b -> {
+            b.put(0, (byte) ProtocolVersion.V6.asInt());     // REQUEST direction, V6 (beta)
+            b.put(1, (byte) (b.get(1) & ~betaBit));          // clear USE_BETA
+        });
+
+        Predicate<Envelope.Header> responseMatcher =
+        h -> (shouldError.test(h.streamId) && h.type == Message.Type.ERROR) || h.type == Message.Type.RESULT;
+
+        ServerConfigurator configurator = ServerConfigurator.builder()
+                                                            .withConsumer(consumer)
+                                                            .withAllocationObserver(observer)
+                                                            .withDecoder(decoder)
+                                                            .build();
+
+        runTest(configurator, codec, messageCount, envelopeProvider, responseMatcher, observer.verifier());
+    }
+
     @Test
     public void testUnrecoverableEnvelopeDecodingErrors()
     {
@@ -551,6 +674,19 @@ public class CQLConnectionTest
         buf.setByte(index, buf.getByte(index) ^ (1 << 4));
     }
 
+    private static class StallingOutboundHandler extends ChannelOutboundHandlerAdapter
+    {
+        volatile ChannelPromise writePromise;
+
+        @Override
+        public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
+        {
+            // Capture the write but neither complete nor forward it: simulates a socket that cannot drain
+            // synchronously. The handler's own finally releases the payload, so we don't touch msg.
+            writePromise = promise;
+        }
+    }
+
     private static class MutableEnvelope extends Envelope
     {
         public MutableEnvelope(Envelope source)
@@ -610,7 +746,6 @@ public class CQLConnectionTest
 
             Message.Request request = new OptionsMessage();
             request.setSource(source);
-            request.setStreamId(source.header.streamId);
             Connection connection = channel.attr(Connection.attributeKey).get();
             request.attach(connection);
 
@@ -629,17 +764,17 @@ public class CQLConnectionTest
         TestConsumer(Message.Response fixedResponse, FrameEncoder frameEncoder)
         {
             this.fixedResponse = fixedResponse;
-            this.responseTemplate = fixedResponse.encode(ProtocolVersion.V5);
+            this.responseTemplate = fixedResponse.encode(ProtocolVersion.V5, 0);
             this.frameEncoder = frameEncoder;
         }
 
-        public void dispatch(Channel channel, Message.Request message, Dispatcher.FlushItemConverter toFlushItem, Overload backpressure)
+        public <P> void dispatch(Channel channel, Message.Request message, Dispatcher.FlushItemConverter<P> toFlushItem, P param, Overload backpressure)
         {
             if (flusher == null)
                 flusher = new SimpleClient.SimpleFlusher(frameEncoder);
 
             Envelope response = Envelope.create(responseTemplate.header.type,
-                                                message.getStreamId(),
+                                                message.getSource().header.streamId,
                                                 ProtocolVersion.V5,
                                                 responseTemplate.header.flags,
                                                 responseTemplate.body.copy());
@@ -648,7 +783,7 @@ public class CQLConnectionTest
             // and flush them to the outbound pipeline
             flusher.schedule(channel.pipeline().lastContext());
             // this simulates the release of the allocated resources that a real flusher would do
-            Flusher.FlushItem.Framed item = (Flusher.FlushItem.Framed)toFlushItem.toFlushItem(channel, message, fixedResponse);
+            Flusher.FlushItem.Framed item = (Flusher.FlushItem.Framed)toFlushItem.toFlushItem(param, channel, message, fixedResponse);
             item.release();
         }
 
@@ -955,6 +1090,21 @@ public class CQLConnectionTest
         }
     }
 
+    /**
+     * Simple adaptor to plug CQL message encoding into pre-V5 pipelines
+     */
+    @ChannelHandler.Sharable
+    public static class ProtocolEncoder extends MessageToMessageEncoder<Message>
+    {
+        public static final ProtocolEncoder instance = new ProtocolEncoder();
+        private ProtocolEncoder(){}
+        public void encode(ChannelHandlerContext ctx, Message source, List<Object> results)
+        {
+            ProtocolVersion version = getConnectionVersion(ctx);
+            results.add(source.encode(version, 0));
+        }
+    }
+
     static class Client
     {
         private final Codec codec;
@@ -993,7 +1143,7 @@ public class CQLConnectionTest
                     ChannelPipeline pipeline = channel.pipeline();
                     // Outbound handlers to enable us to send the initial STARTUP
                     pipeline.addLast("envelopeEncoder", Envelope.Encoder.instance);
-                    pipeline.addLast("messageEncoder", PreV5Handlers.ProtocolEncoder.instance);
+                    pipeline.addLast("messageEncoder", ProtocolEncoder.instance);
                     pipeline.addLast("envelopeDecoder", new Envelope.Decoder());
                     // Inbound handler to perform the handshake & modify the pipeline on receipt of a READY
                     pipeline.addLast("handshake", new MessageToMessageDecoder<Envelope>()
@@ -1071,6 +1221,17 @@ public class CQLConnectionTest
                             flusher.schedule(channel.pipeline().lastContext());
                             ready.countDown();
                         }
+
+                        @Override
+                        public void channelInactive(ChannelHandlerContext ctx)
+                        {
+                            // If the server closes the connection during negotiation (e.g. an unroutable
+                            // protocol error that cannot be answered with a stream id), unblock connect()
+                            // so the test observes the disconnection rather than hanging for a READY/ERROR.
+                            connected = false;
+                            ready.countDown();
+                            ctx.fireChannelInactive();
+                        }
                     });
                 }
             });

test/unit/org/apache/cassandra/transport/ErrorMessageTest.java

@@ -67,7 +67,7 @@ public class ErrorMessageTest extends EncodeAndDecodeTestBase<ErrorMessage>
         boolean dataPresent = false;
         ReadFailureException rfe = new ReadFailureException(consistencyLevel, receivedBlockFor, receivedBlockFor, dataPresent, failureReasonMap1);
 
-        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(rfe), ProtocolVersion.V5);
+        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromTransportException(rfe), ProtocolVersion.V5);
         ReadFailureException deserializedRfe = (ReadFailureException) deserialized.error;
 
         assertEquals(failureReasonMap1, deserializedRfe.failureReasonByEndpoint);
@@ -85,7 +85,7 @@ public class ErrorMessageTest extends EncodeAndDecodeTestBase<ErrorMessage>
         WriteType writeType = WriteType.SIMPLE;
         WriteFailureException wfe = new WriteFailureException(consistencyLevel, receivedBlockFor, receivedBlockFor, writeType, failureReasonMap2);
 
-        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(wfe), ProtocolVersion.V5);
+        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromTransportException(wfe), ProtocolVersion.V5);
         WriteFailureException deserializedWfe = (WriteFailureException) deserialized.error;
 
         assertEquals(failureReasonMap2, deserializedWfe.failureReasonByEndpoint);
@@ -103,7 +103,7 @@ public class ErrorMessageTest extends EncodeAndDecodeTestBase<ErrorMessage>
         ConsistencyLevel consistencyLevel = ConsistencyLevel.SERIAL;
         CasWriteTimeoutException ex = new CasWriteTimeoutException(WriteType.CAS, consistencyLevel, 0, receivedBlockFor, contentions);
 
-        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(ex), ProtocolVersion.V5);
+        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromTransportException(ex), ProtocolVersion.V5);
         assertTrue(deserialized.error instanceof CasWriteTimeoutException);
         CasWriteTimeoutException deserializedEx = (CasWriteTimeoutException) deserialized.error;
 
@@ -124,7 +124,7 @@ public class ErrorMessageTest extends EncodeAndDecodeTestBase<ErrorMessage>
         ConsistencyLevel consistencyLevel = ConsistencyLevel.SERIAL;
         CasWriteTimeoutException ex = new CasWriteTimeoutException(WriteType.CAS, consistencyLevel, receivedBlockFor, receivedBlockFor, contentions);
 
-        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(ex), ProtocolVersion.V4);
+        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromTransportException(ex), ProtocolVersion.V4);
         assertTrue(deserialized.error instanceof WriteTimeoutException);
         assertFalse(deserialized.error instanceof CasWriteTimeoutException);
         WriteTimeoutException deserializedEx = (WriteTimeoutException) deserialized.error;
@@ -142,7 +142,7 @@ public class ErrorMessageTest extends EncodeAndDecodeTestBase<ErrorMessage>
         ConsistencyLevel consistencyLevel = ConsistencyLevel.SERIAL;
         CasWriteUnknownResultException ex = new CasWriteUnknownResultException(consistencyLevel, receivedBlockFor, receivedBlockFor);
 
-        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(ex), ProtocolVersion.V5);
+        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromTransportException(ex), ProtocolVersion.V5);
         assertTrue(deserialized.error instanceof CasWriteUnknownResultException);
         CasWriteUnknownResultException deserializedEx = (CasWriteUnknownResultException) deserialized.error;
 
@@ -160,7 +160,7 @@ public class ErrorMessageTest extends EncodeAndDecodeTestBase<ErrorMessage>
         ConsistencyLevel consistencyLevel = ConsistencyLevel.SERIAL;
         CasWriteUnknownResultException ex = new CasWriteUnknownResultException(consistencyLevel, receivedBlockFor, receivedBlockFor);
 
-        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromException(ex), ProtocolVersion.V4);
+        ErrorMessage deserialized = encodeThenDecode(ErrorMessage.fromTransportException(ex), ProtocolVersion.V4);
         assertTrue(deserialized.error instanceof WriteTimeoutException);
         assertFalse(deserialized.error instanceof CasWriteUnknownResultException);
         WriteTimeoutException deserializedEx = (WriteTimeoutException) deserialized.error;

test/unit/org/apache/cassandra/transport/MessageDispatcherTest.java

@@ -156,7 +156,7 @@ public class MessageDispatcherTest
     public long tryAuth(Callable<Long> check, Message.Request request) throws Exception
     {
         long start = check.call();
-        dispatch.dispatch(null, request, (channel,req,response) -> null, ClientResourceLimits.Overload.NONE);
+        dispatch.dispatch(null, request, (p, channel, req, response) -> null, null, ClientResourceLimits.Overload.NONE);
 
         // While this is timeout based, we should be *well below* a full second on any of this processing in any sane environment.
         long timeout = System.currentTimeMillis();
@@ -175,11 +175,12 @@ public class MessageDispatcherTest
         }
 
         @Override
-        void processRequest(Channel channel,
-                            Message.Request request,
-                            FlushItemConverter forFlusher,
-                            ClientResourceLimits.Overload backpressure,
-                            RequestTime requestTime)
+        <P> void processRequest(Channel channel,
+                                Message.Request request,
+                                FlushItemConverter<P> forFlusher,
+                                P param,
+                                ClientResourceLimits.Overload backpressure,
+                                RequestTime requestTime)
         {
             // noop
         }

test/unit/org/apache/cassandra/transport/ProtocolErrorTest.java

@@ -26,6 +26,7 @@ import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -71,7 +72,12 @@ public class ProtocolErrorTest {
         try {
             dec.decode(null, buf, results);
             Assert.fail("Expected protocol error");
-        } catch (ProtocolException e) {
+        } catch (ErrorMessage.WrappedException e) {
+            // The stream id is recovered using the attempted version's header layout (CASSANDRA-21508):
+            // v1/v2 use a single-byte stream id (header byte 2, which is 0x00 here), whereas v3+ use a
+            // two-byte id (0x0001). See decodeRecoversSingleByteStreamIdForOldProtocolVersions.
+            int expectedStreamId = version < ProtocolVersion.V3.asInt() ? 0x00 : 0x01;
+            Assert.assertEquals(expectedStreamId, e.getStreamId());
             Assert.assertTrue(e.getMessage().contains("Invalid or unsupported protocol version"));
         }
     }
@@ -99,6 +105,108 @@ public class ProtocolErrorTest {
         }
     }
 
+    @Test
+    public void testIncompleteHeaderWithInvalidProtocolVersion() throws Exception
+    {
+        // CASSANDRA-21508: when fewer than a full header's worth of bytes have arrived AND the protocol
+        // version is unsupported, decode cannot trust/recover a stream id. It defers the protocol error and
+        // wraps it with Message.UNSET_STREAM_ID, so the channel-level exception handler closes the connection
+        // rather than emit an unroutable error frame. Exercises Envelope.Decoder.decode() lines 387-398.
+        Envelope.Decoder dec = new Envelope.Decoder();
+
+        List<Object> results = new ArrayList<>();
+        // Unsupported version (two above CURRENT) with only part of a header - fewer than Header.LENGTH (9) bytes.
+        byte[] bytes = new byte[] {
+                (byte) REQUEST.addToVersion(ProtocolVersion.CURRENT.asInt() + 2),  // direction & unsupported version
+                0x00,        // flags
+                0x00, 0x2a,  // partial header, truncated before the length field
+        };
+        ByteBuf buf = Unpooled.wrappedBuffer(bytes);
+        try {
+            dec.decode(null, buf, results);
+            Assert.fail("Expected protocol error");
+        } catch (ProtocolException e) {
+            Assert.assertTrue(e.getMessage().contains("Invalid or unsupported protocol version"));
+        }
+    }
+
+    @Test
+    public void extractHeaderReturnsRecoverableErrorOnBetaFlagViolation() throws Exception
+    {
+        // CASSANDRA-21508: on the framed (V5+) path, a V6 (beta) envelope whose USE_BETA flag is
+        // unset is an invalid but *recoverable* protocol error. Envelope.Decoder.extractHeader documents that
+        // it never throws, and instead returns a HeaderExtractionResult carrying the frame's stream id, so the
+        // caller (CQLMessageHandler.processOneContainedMessage) can route an ERROR back on that stream and keep
+        // processing subsequent envelopes.
+        Envelope.Decoder dec = new Envelope.Decoder();
+
+        int streamId = 42;
+        // A complete 9-byte header advertising V6 (beta) as a REQUEST, with USE_BETA deliberately unset.
+        byte[] header = new byte[] {
+                (byte) REQUEST.addToVersion(ProtocolVersion.V6.asInt()),  // direction & beta version
+                0x00,        // flags - USE_BETA deliberately unset
+                0x00, 0x2a,  // stream id = 42
+                0x05,        // opcode = OPTIONS
+                0x00, 0x00, 0x00, 0x00,  // body length = 0
+        };
+        ByteBuffer buf = ByteBuffer.wrap(header);
+
+        Envelope.Decoder.HeaderExtractionResult result;
+        try
+        {
+            result = dec.extractHeader(buf);
+        }
+        catch (ErrorMessage.WrappedException e)
+        {
+            throw new AssertionError("extractHeader must not throw on a USE_BETA violation; a recoverable " +
+                                     "protocol error escaped as a WrappedException and would tear down the " +
+                                     "connection instead of returning a routable error", e);
+        }
+
+        Assert.assertFalse("A USE_BETA violation should be reported as a (recoverable) extraction error",
+                           result.isSuccess());
+        Assert.assertEquals("the frame's stream id must be preserved so the error can be routed back",
+                            streamId, result.streamId());
+        Assert.assertTrue("expected a USE_BETA protocol error, got: " + result.error().getMessage(),
+                          result.error().getMessage().contains("USE_BETA flag is unset"));
+    }
+
+    @Test
+    public void decodeRecoversSingleByteStreamIdForOldProtocolVersions() throws Exception
+    {
+        // CASSANDRA-21508: protocol versions 1 and 2 use a shorter header whose stream id is a single
+        // byte. When such a version is rejected, decode must recover the stream id using that layout.
+        // Reading a 16-bit value would splice the stream id byte together with the following opcode byte and
+        // recover a bogus id (e.g. stream 0 + STARTUP opcode 1 -> stream 1), routing the error to a stream the
+        // client never used. A v1/v2 client (which downgrades on such an error) would then never see it and
+        // its connection would time out - as observed in ProtocolNegotiationTest#olderVersionsAreUnsupported.
+        Envelope.Decoder dec = new Envelope.Decoder();
+
+        int streamId = 0x07;
+        List<Object> results = new ArrayList<>();
+        // A full (>= Header.LENGTH bytes) v1 frame so decode reaches the stream-id recovery, not the
+        // incomplete-header path: single-byte stream id, STARTUP opcode, and one body byte.
+        byte[] bytes = new byte[] {
+                (byte) REQUEST.addToVersion(1),  // unsupported v1
+                0x00,                            // flags
+                (byte) streamId,                 // v1/v2 single-byte stream id
+                0x01,                            // opcode = STARTUP (would be spliced in as the low byte)
+                0x00, 0x00, 0x00, 0x01,          // body length = 1
+                0x00                             // 1 body byte, so readableBytes == Header.LENGTH
+        };
+        ByteBuf buf = Unpooled.wrappedBuffer(bytes);
+        try {
+            dec.decode(null, buf, results);
+            Assert.fail("Expected protocol error");
+        } catch (ErrorMessage.WrappedException e) {
+            Assert.assertEquals("a v1/v2 stream id must be read as a single byte, not spliced with the opcode",
+                                streamId, e.getStreamId());
+            Assert.assertTrue("expected a ProtocolException cause, got: " + e.getCause(),
+                              e.getCause() instanceof ProtocolException);
+            Assert.assertTrue(e.getMessage().contains("Invalid or unsupported protocol version"));
+        }
+    }
+
     @Test
     public void testInvalidDirection() throws Exception
     {
@@ -159,7 +267,7 @@ public class ProtocolErrorTest {
     public void testErrorMessageWithNullString()
     {
         // test for CASSANDRA-11167
-        ErrorMessage msg = ErrorMessage.fromException(new ServerError((String) null));
+        ErrorMessage msg = ErrorMessage.fromTransportException(new ServerError((String) null));
         assert msg.toString().endsWith("null") : msg.toString();
         int size = ErrorMessage.codec.encodedSize(msg, ProtocolVersion.CURRENT);
         ByteBuf buf = Unpooled.buffer(size);

test/unit/org/apache/cassandra/transport/ProtocolNegotiationTest.java

@@ -125,29 +125,31 @@ public class ProtocolNegotiationTest extends CQLTester
             for (int i = 0; i < 100; i++)
             {
                 int streamId = random.nextInt(254) + 1;
-                options.setStreamId(streamId);
+                options.setSource(new Envelope(Envelope.Header.dummy(streamId, options.type), null));
                 Message.Response response = client.execute(options);
+                // The stream id is stamped onto the frame at encoding time; verify it round-trips by
+                // reading it back off the response's source envelope.
                 assertEquals(String.format("StreamId mismatch; version: %s, seed: %s, iter: %s, expected: %s, actual: %s",
-                                           version, seed, i, streamId, response.getStreamId()),
-                             streamId, response.getStreamId());
+                                           version, seed, i, streamId, response.getSource().header.streamId),
+                             streamId, response.getSource().header.streamId);
             }
 
             int streamId = random.nextInt(254) + 1;
             // STARTUP messages are handled by the initial connection handler
             StartupMessage startup = new StartupMessage(ImmutableMap.of(CQL_VERSION, QueryProcessor.CQL_VERSION.toString()));
-            startup.setStreamId(streamId);
+            startup.setSource(new Envelope(Envelope.Header.dummy(streamId, startup.type), null));
             Message.Response response = client.execute(startup);
             assertEquals(String.format("StreamId mismatch after negotiation; version: %s, expected: %s, actual %s",
-                                       version, streamId, response.getStreamId()),
-                         streamId, response.getStreamId());
+                                       version, streamId, response.getSource().header.streamId),
+                         streamId, response.getSource().header.streamId);
 
             // Following STARTUP, the version specific handlers are fully responsible for processing messages
             QueryMessage query = new QueryMessage("SELECT * FROM system.local", QueryOptions.DEFAULT);
-            query.setStreamId(streamId);
+            query.setSource(new Envelope(Envelope.Header.dummy(streamId, query.type), null));
             response = client.execute(query);
             assertEquals(String.format("StreamId mismatch after negotiation; version: %s, expected: %s, actual %s",
-                                       version, streamId, response.getStreamId()),
-                         streamId, response.getStreamId());
+                                       version, streamId, response.getSource().header.streamId),
+                         streamId, response.getSource().header.streamId);
         }
         catch (IOException e)
         {
@@ -214,9 +216,9 @@ public class ProtocolNegotiationTest extends CQLTester
             QueryMessage query = new QueryMessage("SELECT * FROM system.local", QueryOptions.DEFAULT)
             {
                 @Override
-                public Envelope encode(ProtocolVersion originalVersion)
+                public Envelope encode(ProtocolVersion originalVersion, int streamId)
                 {
-                    return super.encode(v);
+                    return super.encode(v, streamId);
                 }
             };
             try