mirror of https://github.com/apache/cassandra
SQUASH
run-ci must not silently drop a site's helm values customisations Long-lived instances like pre-ci.cassandra.apache.org carry values absent from jenkins-deployment.yaml, and any `helm upgrade -f jenkins-deployment.yaml` drops them. Keep them in a separate file and pass `--values-override`; helm merges it over the repo's. Before upgrading, compare the deployed values against what is about to be applied and require confirmation for anything only the site holds. This catches added keys and dropped list items; it cannot tell a customisation from a key the repo has since removed, hence a prompt, and it is blind to a key present in both with a locally edited value, an agent.podTemplates entry for instance. Aborts when not on a tty. Also keep the jenkins-home claim on uninstall, so --only-tear-down no longer destroys build history; jenkins-test.sh validates .jenkins/ without a cluster, including the yaml embedded in the pod templates and JCasC scripts that the chart never parses; a workflow runs it and the run-ci tests on changes to either. The requests timeouts and unused wait_for_jenkins_http argument are drive-bys to get run-ci's own lint command clean, so the workflow can enforce it.
This commit is contained in:
mck
parent
2e90bd711a
commit
b6180d8503
|
|
@ -72,7 +72,7 @@
|
|||
<exclude name="test/data/jmxdump/cassandra-*-jmx.yaml"/>
|
||||
<!-- Documentation files -->
|
||||
<exclude name=".github/pull_request_template.md"/>
|
||||
<exclude name=".github/workflows/code-check.yaml"/>
|
||||
<exclude name=".github/workflows/**.yaml"/>
|
||||
<exclude NAME="doc/modules/**/*"/>
|
||||
<exclude NAME="src/java/**/*.md"/>
|
||||
<exclude NAME="**/README*"/>
|
||||
|
|
|
|||
139
.build/run-ci
139
.build/run-ci
|
|
@ -25,7 +25,7 @@ Python dependencies are found in .build/run-ci.d/requirements.txt
|
|||
Custom environment variables can be set in .build/.run-ci.env
|
||||
|
||||
lint with:
|
||||
`pylint --disable=C0301,W0511,C0103,W0702,C0415,C0116,C0115,R0914,W0603,R0915,R0913,R0911 run-ci`
|
||||
`pylint --disable=C0301,W0511,C0103,W0702,C0415,C0116,C0115,R0914,W0603,R0915,R0913,R0917,R0911,W0212,W0621 run-ci`
|
||||
|
||||
test with:
|
||||
`python .build/run-ci.d/run-ci-test.py`
|
||||
|
|
@ -51,9 +51,10 @@ from urllib.request import urlretrieve
|
|||
from typing import Optional, Tuple
|
||||
|
||||
# External Libraries (`pip install -r .build/run-ci.d/requirements.txt`)
|
||||
import requests
|
||||
import yaml
|
||||
from bs4 import BeautifulSoup
|
||||
from kubernetes import client, config, stream
|
||||
import requests
|
||||
|
||||
try:
|
||||
import jenkins
|
||||
|
|
@ -73,9 +74,9 @@ def base_job_name(args) -> str:
|
|||
"""
|
||||
if not hasattr(base_job_name, "_cached_result"):
|
||||
raw_url = args.repository.replace("https://github.com/", "https://raw.githubusercontent.com/").removesuffix(".git") + f"/{args.branch}/build.xml"
|
||||
if 200 != requests.head(raw_url).status_code:
|
||||
if 200 != requests.head(raw_url, timeout=30).status_code:
|
||||
raise ValueError(f"GitHub unavailable, or this branch has not been pushed yet: {args.repository} @ {args.branch} (or remote tracking not setup up: `git config --get branch.{args.branch}.remote` and `git config --get branch.{args.branch}.merge`)")
|
||||
response = requests.get(raw_url)
|
||||
response = requests.get(raw_url, timeout=30)
|
||||
response.raise_for_status()
|
||||
for line in response.text.splitlines():
|
||||
if 'property' in line and 'name="base.version"' in line:
|
||||
|
|
@ -98,7 +99,7 @@ def is_local_git_dirty(args) -> bool:
|
|||
# use base_job_name to verify the remote branch exists
|
||||
base_job_name(args)
|
||||
# check if the working directory is clean
|
||||
clean = subprocess.run(["git", "-C", str(CASSANDRA_DIR), "diff-index", "--quiet", "HEAD", "--"]).returncode
|
||||
clean = subprocess.run(["git", "-C", str(CASSANDRA_DIR), "diff-index", "--quiet", "HEAD", "--"], check=False).returncode
|
||||
# check if there are unpushed committed changes
|
||||
unpushed_commits = bool(subprocess.run(["git", "-C", str(CASSANDRA_DIR), "log", "@{u}..HEAD", "--name-only"],
|
||||
capture_output=True, text=True, check=False).stdout.strip())
|
||||
|
|
@ -191,6 +192,7 @@ def argument_parser() -> argparse.ArgumentParser:
|
|||
parser.add_argument("-k", "--dtest-branch", default=DEFAULT_DTEST_REPO_BRANCH, help="DTest repository branch.")
|
||||
parser.add_argument("-s", "--setup", action="store_true", help="Set up Jenkins before the build.")
|
||||
parser.add_argument("--only-setup", action="store_true", help="Only install Jenkins into the k8s cluster.")
|
||||
parser.add_argument("-f", "--values-override", help="Path to an additional helm values file, applied over .jenkins/k8s/jenkins-deployment.yaml. Required when the target cluster carries site customisations, see .jenkins/k8s/README.md")
|
||||
parser.add_argument("--tear-down", action="store_true", help="Tear down Jenkins after the build.")
|
||||
parser.add_argument("--only-tear-down", action="store_true", help="Only tear down Jenkins.")
|
||||
parser.add_argument("--only-node-cleaner", action="store_true", help="Only run the node cleaner. The node cleaner scans the k8s nodes, eagerly terminating those unused.")
|
||||
|
|
@ -211,6 +213,8 @@ def parse_arguments() -> argparse.Namespace:
|
|||
assert not (args.setup and args.only_setup), "Both --setup or --only-setup cannot be specified."
|
||||
assert not (args.tear_down and args.only_tear_down), "Both --tear-down or --only-tear-down cannot be specified."
|
||||
assert not ("custom" == args.profile and not args.profile_custom_regexp), "Custom profile requires --profile-custom-regexp."
|
||||
assert not (args.values_override and not (args.setup or args.only_setup)), "--values-override requires --setup or --only-setup."
|
||||
assert not (args.values_override and not Path(args.values_override).is_file()), f"No such values override file: {args.values_override}"
|
||||
|
||||
if not args.url and os.environ.get("JENKINS_URL"):
|
||||
args.url = os.environ.get("JENKINS_URL")
|
||||
|
|
@ -252,19 +256,113 @@ def run_kubectl_command(kubeconfig: Optional[str], kubecontext: Optional[str], k
|
|||
cmd += command
|
||||
return subprocess.run(cmd, capture_output=True, text=True, check=True).stdout.strip()
|
||||
|
||||
def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ns: str):
|
||||
"""Installs Jenkins Operator using Helm in the specified K8s namespace."""
|
||||
print("Adding Helm repository for Jenkins Operator...")
|
||||
subprocess.run(["helm", "repo", "add", "jenkins", "https://charts.jenkins.io"], check=True)
|
||||
subprocess.run(["helm", "repo", "update"], check=True)
|
||||
|
||||
def run_helm_command(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ns: str, command: list,
|
||||
capture_output: bool = True, check: bool = True) -> subprocess.CompletedProcess:
|
||||
"""Runs a helm command with the specified kubeconfig, context and namespace."""
|
||||
cmd = ["helm"]
|
||||
if kubeconfig:
|
||||
cmd += ["--kubeconfig", kubeconfig]
|
||||
if kubecontext:
|
||||
cmd += ["--kube-context", kubecontext]
|
||||
cmd += ["--namespace", kube_ns, "upgrade", "--install", "-f", DEPLOY_YAML, "cassius", "jenkins/jenkins", "--wait"]
|
||||
result = subprocess.run(cmd, capture_output=True, check=True)
|
||||
cmd += ["--namespace", kube_ns]
|
||||
cmd += command
|
||||
return subprocess.run(cmd, capture_output=capture_output, text=True, check=check)
|
||||
|
||||
def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_ns: str,
|
||||
values_override: Optional[str] = None):
|
||||
"""Installs Jenkins Operator using Helm in the specified K8s namespace."""
|
||||
|
||||
def confirm_helm_updates():
|
||||
"""Prompts before an upgrade drops any values the deployed jenkins currently has."""
|
||||
|
||||
def helm_values() -> dict:
|
||||
"""
|
||||
The values a deployed jenkins was last installed with, or an empty dict when there is no release.
|
||||
These are the user-supplied values, whatever files they came from, so they include any customisations
|
||||
made to the site outside of `.jenkins/k8s/jenkins-deployment.yaml`.
|
||||
"""
|
||||
result = run_helm_command(kubeconfig, kubecontext, kube_ns,
|
||||
["get", "values", "cassius", "-o", "yaml"], check=False)
|
||||
if result.returncode != 0:
|
||||
debug(f"No existing cassius release found in namespace {kube_ns}: {result.stderr.strip()}")
|
||||
return {}
|
||||
return yaml.safe_load(result.stdout) or {}
|
||||
|
||||
def merge_values(base: dict, override: dict) -> dict:
|
||||
"""Merges two helm values files the way helm does: maps key by key, everything else replaced."""
|
||||
merged = dict(base)
|
||||
for key, value in override.items():
|
||||
if isinstance(value, dict) and isinstance(merged.get(key), dict):
|
||||
merged[key] = merge_values(merged[key], value)
|
||||
else:
|
||||
merged[key] = value
|
||||
return merged
|
||||
|
||||
def detect_lost_values(live: dict, proposed: dict) -> dict:
|
||||
"""
|
||||
Values the deployed jenkins has that an upgrade would drop, as {dotted.key.path: live value}.
|
||||
|
||||
A key held live but absent from what is about to be applied is either a customisation made to this site,
|
||||
or a key that `.jenkins/k8s/jenkins-deployment.yaml` has removed since the site was last deployed.
|
||||
Note also what this cannot see: a key that exists in both but was given a different value locally, such
|
||||
as an edited `agent.podTemplates` entry, is silently overwritten.
|
||||
Read the diff of `helm template` before deploying an unfamiliar site.
|
||||
"""
|
||||
def leaf_values(values, path: str = "") -> dict:
|
||||
"""Flattens a values map to {dotted.key.path: value}, lists are leaves (helm replaces them)."""
|
||||
if not isinstance(values, dict):
|
||||
return {path: values}
|
||||
leaves = {}
|
||||
for key, value in values.items():
|
||||
leaves.update(leaf_values(value, f"{path}.{key}" if path else str(key)))
|
||||
return leaves
|
||||
|
||||
live_leaves, proposed_leaves = leaf_values(live), leaf_values(proposed)
|
||||
lost = {path: value for path, value in live_leaves.items() if path not in proposed_leaves}
|
||||
# lists are replaced wholesale, so also report items dropped from a list that is otherwise still there
|
||||
for path, value in live_leaves.items():
|
||||
if isinstance(value, list) and isinstance(proposed_leaves.get(path), list):
|
||||
dropped = [item for item in value if item not in proposed_leaves[path]]
|
||||
if dropped:
|
||||
lost[f"{path}[]"] = dropped
|
||||
return lost
|
||||
|
||||
with open(DEPLOY_YAML, encoding="utf-8") as deploy_yaml:
|
||||
proposed = yaml.safe_load(deploy_yaml) or {}
|
||||
if values_override:
|
||||
with open(values_override, encoding="utf-8") as override_yaml:
|
||||
proposed = merge_values(proposed, yaml.safe_load(override_yaml) or {})
|
||||
|
||||
lost = detect_lost_values(helm_values(), proposed)
|
||||
if not lost:
|
||||
return
|
||||
|
||||
print(f"\nWARNING: {len(lost)} value(s) the deployed jenkins has are absent from what is about to be applied.")
|
||||
print("Each is either a customisation of this site, or a key removed from jenkins-deployment.yaml since"
|
||||
" the site was last deployed. Upgrading drops them:\n")
|
||||
for path in sorted(lost):
|
||||
value = str(lost[path]).replace("\n", " ")
|
||||
print(f" {path}: {value[:100] + '…' if len(value) > 100 else value}")
|
||||
print(f"\nTo keep any of them, add them to a values override file (see .jenkins/k8s/README.md) and pass"
|
||||
f" `--values-override`{' (the file passed does not contain them)' if values_override else ''}.")
|
||||
|
||||
if not sys.stdin.isatty():
|
||||
print("Refusing to drop them when running non-interactively.")
|
||||
sys.exit(1)
|
||||
if input("\nDrop these values and continue? [y/N] ").strip().lower() not in ("y", "yes"):
|
||||
print("Aborted, nothing was deployed.")
|
||||
sys.exit(1)
|
||||
|
||||
confirm_helm_updates()
|
||||
|
||||
print("Adding Helm repository for Jenkins Operator...")
|
||||
subprocess.run(["helm", "repo", "add", "jenkins", "https://charts.jenkins.io"], check=True)
|
||||
subprocess.run(["helm", "repo", "update"], check=True)
|
||||
|
||||
# site customisations are applied last, helm merges each -f over the previous
|
||||
values_files = ["-f", DEPLOY_YAML] + (["-f", values_override] if values_override else [])
|
||||
result = run_helm_command(kubeconfig, kubecontext, kube_ns,
|
||||
["upgrade", "--install"] + values_files + ["cassius", "jenkins/jenkins", "--wait"])
|
||||
|
||||
run_kubectl_command(kubeconfig, kubecontext, kube_ns,
|
||||
["exec", DEFAULT_POD_NAME, "--",
|
||||
|
|
@ -276,7 +374,7 @@ def install_jenkins(kubeconfig: Optional[str], kubecontext: Optional[str], kube_
|
|||
sys.exit(1)
|
||||
|
||||
|
||||
def wait_for_jenkins_http(ip: str, timeout: int = 300):
|
||||
def wait_for_jenkins_http(ip: str):
|
||||
host, port = (ip.rsplit(":", 1)[0], int(ip.rsplit(":", 1)[1])) if ":" in ip else (ip, 80)
|
||||
spin_while(f"Waiting for Jenkins HTTP at {host}:{port}… ", lambda: _tcp_connect_ok(host, port))
|
||||
|
||||
|
|
@ -794,13 +892,10 @@ def cleanup_and_maybe_teardown(kubeconfig: Optional[str], kubecontext: Optional[
|
|||
IS_RUNNING = False
|
||||
if tear_down:
|
||||
print("Cleaning up Jenkins and all resources.")
|
||||
cmd = ["helm"]
|
||||
if kubeconfig:
|
||||
cmd += ["--kubeconfig", kubeconfig]
|
||||
if kubecontext:
|
||||
cmd += ["--kube-context", kubecontext]
|
||||
cmd += ["--namespace", kube_ns, "uninstall", "cassius"]
|
||||
subprocess.run(cmd, check=True)
|
||||
run_helm_command(kubeconfig, kubecontext, kube_ns, ["uninstall", "cassius"], capture_output=False)
|
||||
# the pvc is annotated `helm.sh/resource-policy: keep`, see .jenkins/k8s/jenkins-deployment.yaml
|
||||
print(f"Jenkins uninstalled. The jenkins-home volume was kept, delete it with:\n"
|
||||
f" kubectl --namespace {kube_ns} delete pvc cassius-jenkins")
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
|
@ -838,7 +933,7 @@ def main():
|
|||
if args.setup or args.only_setup:
|
||||
init_k8s_namespace(k8s_client, DEFAULT_KUBE_NS)
|
||||
with helm_installation_lock(Path("/tmp/.cassandra-run-ci.lock")):
|
||||
install_jenkins(args.kubeconfig, args.kubecontext, DEFAULT_KUBE_NS)
|
||||
install_jenkins(args.kubeconfig, args.kubecontext, DEFAULT_KUBE_NS, args.values_override)
|
||||
|
||||
(ip, server) = get_jenkins(k8s_client, args, DEFAULT_KUBE_NS)
|
||||
if args.setup or args.only_setup:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
```
|
||||
➤ .build/run-ci --help
|
||||
usage: run-ci [-h] [-c KUBECONFIG] [-x KUBECONTEXT] [-i URL] [-u USER] [-r REPOSITORY] [-b BRANCH] [-p {packaging,skinny,pre-commit,pre-commit w/ upgrades,post-commit,custom}] [-e PROFILE_CUSTOM_REGEXP] [-j JDK] [-d DTEST_REPOSITORY] [-k DTEST_BRANCH]
|
||||
[-s] [--only-setup] [--tear-down] [--only-tear-down] [--only-node-cleaner] [-o DOWNLOAD_RESULTS]
|
||||
[-s] [--only-setup] [-v VALUES_OVERRIDE] [--tear-down] [--only-tear-down] [--only-node-cleaner] [-o DOWNLOAD_RESULTS]
|
||||
|
||||
Run CI pipeline for Cassandra on K8s using Jenkins.
|
||||
|
||||
|
|
@ -30,6 +30,8 @@ options:
|
|||
DTest repository branch.
|
||||
-s, --setup Set up Jenkins before the build.
|
||||
--only-setup Only install Jenkins into the k8s cluster.
|
||||
-v VALUES_OVERRIDE, --values-override VALUES_OVERRIDE
|
||||
Path to an additional helm values file, applied over .jenkins/k8s/jenkins-deployment.yaml. Required when the target cluster carries site customisations, see .jenkins/k8s/README.md
|
||||
--tear-down Tear down Jenkins after the build.
|
||||
--only-tear-down Only tear down Jenkins.
|
||||
--only-node-cleaner Only run the node cleaner. The node cleaner scans the k8s nodes, eagerly terminating those unused.
|
||||
|
|
@ -63,7 +65,15 @@ Setup/Update Jenkins Helm into your current kubeconfig
|
|||
.build/run-ci --only-setup
|
||||
```
|
||||
|
||||
Uninstall Jenkins from your current kubeconfig
|
||||
Setup/Update Jenkins Helm into a cluster that carries site customisations, e.g. pre-ci.cassandra.apache.org
|
||||
```
|
||||
.build/run-ci --only-setup --values-override ~/.cassandra-ci/pre-ci-overrides.yaml
|
||||
```
|
||||
|
||||
Before any setup, the values already deployed are compared against those about to be applied. Any value the deployed jenkins holds that the new files lack is listed, and confirmation is asked for before it is dropped; running non-interactively aborts instead. See `.jenkins/k8s/README.md` for what this can and cannot catch.
|
||||
|
||||
Uninstall Jenkins from your current kubeconfig.
|
||||
```
|
||||
.build/run-ci --only-tear-down
|
||||
```
|
||||
```
|
||||
The jenkins-home volume is kept; delete it separately with `kubectl delete pvc cassius-jenkins`
|
||||
|
|
@ -18,6 +18,7 @@ bs4
|
|||
dotenv
|
||||
kubernetes
|
||||
python-jenkins
|
||||
pyyaml
|
||||
requests
|
||||
|
||||
# optional for different clouds
|
||||
|
|
|
|||
|
|
@ -27,12 +27,14 @@
|
|||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
# Import the functions from the script
|
||||
from run_ci import (
|
||||
DEPLOY_YAML,
|
||||
debug,
|
||||
install_jenkins,
|
||||
get_jenkins,
|
||||
|
|
@ -55,13 +57,84 @@ class TestCIPipeline(unittest.TestCase):
|
|||
debug("Test message")
|
||||
mock_print.assert_called_with("Test message")
|
||||
|
||||
# the pre-flight check is nested inside install_jenkins, so it is exercised through it: the mocked
|
||||
# `helm get values` stdout stands in for the values the site already has deployed
|
||||
LIVE_STORAGE_CLASS = "persistence:\n storageClass: gp2\n"
|
||||
|
||||
@patch('run_ci.subprocess.run')
|
||||
def test_install_jenkins(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0)
|
||||
# empty stdout, i.e. nothing deployed yet, so the pre-flight check has nothing to warn about
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout="")
|
||||
install_jenkins("test-namespace", Path("/fake/cassandra/dir"), "default")
|
||||
mock_run.assert_any_call(["helm", "repo", "add", "jenkins", "https://charts.jenkins.io"], check=True)
|
||||
mock_run.assert_any_call(["helm", "repo", "update"], check=True)
|
||||
|
||||
@patch('run_ci.subprocess.run')
|
||||
def test_install_jenkins_values_override(self, mock_run):
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout="")
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".yaml") as override:
|
||||
override.write(self.LIVE_STORAGE_CLASS)
|
||||
override.flush()
|
||||
install_jenkins(None, None, "default", override.name)
|
||||
upgrade_cmd = [c for c in [call.args[0] for call in mock_run.call_args_list] if "upgrade" in c][0]
|
||||
# the site's overrides must come after, and never replace, the repo's deployment yaml
|
||||
self.assertEqual(["-f", DEPLOY_YAML, "-f", override.name], upgrade_cmd[5:9])
|
||||
|
||||
@patch('run_ci.sys.stdin.isatty')
|
||||
@patch('run_ci.print')
|
||||
@patch('run_ci.subprocess.run')
|
||||
def test_install_jenkins_aborts_non_interactively(self, mock_run, mock_print, mock_isatty):
|
||||
mock_isatty.return_value = False
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout=self.LIVE_STORAGE_CLASS)
|
||||
with self.assertRaises(SystemExit):
|
||||
install_jenkins(None, None, "default")
|
||||
self.assertEqual([], [c for c in [call.args[0] for call in mock_run.call_args_list] if "upgrade" in c])
|
||||
# and continues when the customisation is passed back in as an override. This also proves the merge is
|
||||
# per key: were the override to replace the whole persistence map, its other keys would now be reported lost
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".yaml") as override:
|
||||
override.write(self.LIVE_STORAGE_CLASS)
|
||||
override.flush()
|
||||
install_jenkins(None, None, "default", override.name)
|
||||
self.assertEqual(1, len([c for c in [call.args[0] for call in mock_run.call_args_list] if "upgrade" in c]))
|
||||
|
||||
@patch('run_ci.input')
|
||||
@patch('run_ci.sys.stdin.isatty')
|
||||
@patch('run_ci.print')
|
||||
@patch('run_ci.subprocess.run')
|
||||
def test_install_jenkins_prompts(self, mock_run, mock_print, mock_isatty, mock_input):
|
||||
mock_isatty.return_value = True
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout=self.LIVE_STORAGE_CLASS)
|
||||
mock_input.return_value = "n"
|
||||
with self.assertRaises(SystemExit):
|
||||
install_jenkins(None, None, "default")
|
||||
mock_input.return_value = "y"
|
||||
install_jenkins(None, None, "default")
|
||||
|
||||
@patch('run_ci.sys.stdin.isatty')
|
||||
@patch('run_ci.print')
|
||||
@patch('run_ci.subprocess.run')
|
||||
def test_install_jenkins_reports_only_detectable_losses(self, mock_run, mock_print, mock_isatty):
|
||||
mock_isatty.return_value = False
|
||||
# a plugin only this site installs is reported, as is a key the site alone holds; a key held in both but
|
||||
# locally edited (persistence.size) cannot be seen, and must not be claimed
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout="persistence:\n size: 1Ti\n"
|
||||
"controller:\n installPlugins:\n - site-only-plugin\n")
|
||||
with self.assertRaises(SystemExit):
|
||||
install_jenkins(None, None, "default")
|
||||
printed = " ".join(str(call.args[0]) for call in mock_print.call_args_list if call.args)
|
||||
self.assertIn("controller.installPlugins[]", printed)
|
||||
self.assertIn("site-only-plugin", printed)
|
||||
self.assertNotIn("persistence.size", printed)
|
||||
|
||||
@patch('run_ci.print')
|
||||
@patch('run_ci.subprocess.run')
|
||||
def test_install_jenkins_when_nothing_deployed(self, mock_run, mock_print):
|
||||
# `helm get values` fails when there is no release, and nothing is then warned about
|
||||
mock_run.side_effect = lambda cmd, **kwargs: MagicMock(returncode=1 if "get" in cmd else 0,
|
||||
stdout="", stderr="release: not found")
|
||||
install_jenkins(None, None, "default")
|
||||
self.assertEqual([], [call.args[0] for call in mock_print.call_args_list if "WARNING" in str(call.args)])
|
||||
|
||||
@patch('run_ci.subprocess.run')
|
||||
@patch('run_ci.jenkins.Jenkins')
|
||||
def test_get_jenkins(self, mock_jenkins, mock_run):
|
||||
|
|
@ -91,13 +164,14 @@ class TestCIPipeline(unittest.TestCase):
|
|||
@patch('run_ci.stream.stream')
|
||||
def test_delete_remote_junit_files(self, mock_stream):
|
||||
mock_k8s_client = MagicMock()
|
||||
delete_remote_junit_files(mock_k8s_client, "test-pod", "test-namespace", 456)
|
||||
delete_remote_junit_files(mock_k8s_client, "test-pod", "test-namespace", "test-job", 456)
|
||||
mock_stream.assert_called()
|
||||
|
||||
@patch('run_ci.subprocess.run')
|
||||
def test_cleanup_and_maybe_teardown(self, mock_run):
|
||||
cleanup_and_maybe_teardown(None, None, "test-namespace", True)
|
||||
mock_run.assert_called_with(["helm", "--namespace", "test-namespace", "uninstall", "cassius"], check=True)
|
||||
mock_run.assert_called_with(["helm", "--namespace", "test-namespace", "uninstall", "cassius"],
|
||||
capture_output=False, text=True, check=True)
|
||||
|
||||
@patch('run_ci.fcntl.flock')
|
||||
def test_helm_installation_lock(self, mock_flock):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
name: CI Check
|
||||
|
||||
# Checks the CI tooling itself: the run-ci script and everything under .jenkins/
|
||||
# Nothing here touches a k8s cluster; the helm and groovy steps only render and parse.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- '**'
|
||||
paths:
|
||||
- '.build/run-ci'
|
||||
- '.build/run-ci.d/**'
|
||||
- '.jenkins/**'
|
||||
- '.github/workflows/jenkins-check.yaml'
|
||||
pull_request:
|
||||
paths:
|
||||
- '.build/run-ci'
|
||||
- '.build/run-ci.d/**'
|
||||
- '.jenkins/**'
|
||||
- '.github/workflows/jenkins-check.yaml'
|
||||
|
||||
jobs:
|
||||
run-ci:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Python 3
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.x'
|
||||
- name: Install dependencies
|
||||
run: pip install pylint -r .build/run-ci.d/requirements.txt
|
||||
- name: Lint run-ci
|
||||
run: pylint --disable=C0301,W0511,C0103,W0702,C0415,C0116,C0115,R0914,W0603,R0915,R0913,R0917,R0911,W0212,W0621 .build/run-ci
|
||||
- name: Run run-ci unit tests
|
||||
run: python .build/run-ci.d/run-ci-test.py
|
||||
- name: Check run-ci argument parsing
|
||||
run: python .build/run-ci --help
|
||||
|
||||
jenkins-validate:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
- name: Python 3
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.x'
|
||||
- name: Install dependencies
|
||||
run: pip install pyyaml
|
||||
- name: Helm
|
||||
uses: azure/setup-helm@v4
|
||||
# validate.sh runs groovy from docker when it is not on the PATH
|
||||
- name: Validate .jenkins/
|
||||
run: .jenkins/k8s/jenkins-test.sh
|
||||
|
|
@ -24,14 +24,14 @@ ZONE="us-central1-c"
|
|||
gcloud container clusters create ${CLUSTER_NAME} --machine-type e2-standard-8 --disk-type=pd-ssd --num-nodes 1 --node-labels=cassandra.jenkins.controller=true --autoscaling-profile optimize-utilization --zone ${ZONE}
|
||||
|
||||
# small resource nodes
|
||||
gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE}
|
||||
gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE}
|
||||
|
||||
# medium resource nodes
|
||||
# preference (by cost): n2-highcpu-8, c3-highcpu-8, n4-highcpu-8, n1-highcpu-16
|
||||
gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=100 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE}
|
||||
gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=100 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.medium=true --zone ${ZONE}
|
||||
|
||||
# large resource nodes
|
||||
gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE}
|
||||
gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --disk-size=107 --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE}
|
||||
|
||||
# For each sized resource nodes, pick any machine type that fits, those listed above should work and be the most cost-effective, but this can change region to region
|
||||
# See https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/Jenkinsfile#L35-L38
|
||||
|
|
@ -64,17 +64,81 @@ kubectl exec -it svc/cassius-jenkins -c jenkins -- /bin/cat /run/secrets/additio
|
|||
|
||||
This leaves the controller running, a single e2-standard-8 instance. All other node-pools downscale to zero.
|
||||
|
||||
### Local-only Access
|
||||
## Upgrading an existing instance (e.g. pre-ci.cassandra.apache.org)
|
||||
|
||||
A long-lived site like pre-ci.cassandra.apache.org may carry customisations: hostname, cloud load-balancer, storage class; that are deliberately absent from `jenkins-deployment.yaml`.
|
||||
|
||||
Running `helm upgrade -f jenkins-deployment.yaml` or `.build/run-ci --only-setup` will drop those customisations.
|
||||
|
||||
Instead keep the customisations in separate overrides file, using it like
|
||||
```
|
||||
.build/run-ci --only-setup --values-override <file>
|
||||
```
|
||||
or as a second `-f` argument to `helm upgrade`.
|
||||
|
||||
|
||||
##### To collect overridden values
|
||||
|
||||
To get and diff currently deployed values against the version of `jenkins-deployment.yaml` that is deployed.
|
||||
```
|
||||
RELEASE=cassius
|
||||
NS=default
|
||||
kubectl config current-context # confirm correct context
|
||||
DEPLOYED_COMMIT=cassandra-5.0 # the last jenkins-deployment.yaml deployed git commit sha
|
||||
|
||||
helm get values ${RELEASE} -n ${NS} -o yaml > /tmp/cassandra-ci-live-values.yaml
|
||||
|
||||
git show ${DEPLOYED_COMMIT}:.jenkins/k8s/jenkins-deployment.yaml > /tmp/deployed.yaml
|
||||
|
||||
for f in /tmp/deployed.yaml /tmp/cassandra-ci-live-values.yaml ; do
|
||||
python3 -c 'import sys,yaml;print(yaml.safe_dump(yaml.safe_load(open(sys.argv[1])),sort_keys=True,width=10000))' ${f} > ${f}.sorted
|
||||
done
|
||||
|
||||
diff -u /tmp/deployed.yaml.sorted /tmp/cassandra-ci-live-values.yaml.sorted
|
||||
```
|
||||
|
||||
Copy the genuine customisations you need to keep into `~/.cassandra-ci/<site>-overrides.yaml`, keeping the full key path.
|
||||
|
||||
For example:
|
||||
```
|
||||
controller:
|
||||
ingress:
|
||||
hostName: pre-ci.cassandra.apache.org # note the capital N, `hostname` is silently ignored
|
||||
serviceAnnotations: # AWS load-balancer-controller: static EIP, public subnet
|
||||
service.beta.kubernetes.io/aws-load-balancer-name: pre-ci-apache-cassandra
|
||||
...
|
||||
persistence:
|
||||
storageClass: gp2
|
||||
```
|
||||
|
||||
Beware how Helm merges: maps are merged key by key, but lists and strings are *replaced* wholesale. Each `agent.podTemplates.*` entry is one multi-line string, so overriding a pod template masks every repo-side change to that template. Prefer keeping pod-template customisations out of the overrides file; where that is unavoidable, re-apply the repo's changes to the overridden copy by hand at each upgrade.
|
||||
|
||||
|
||||
### Rolling back a bad Helm upgrade
|
||||
|
||||
```
|
||||
helm history ${RELEASE} -n ${NS}
|
||||
helm rollback ${RELEASE} <last-good-revision> -n ${NS} --wait --timeout 15m
|
||||
```
|
||||
|
||||
`helm rollback` restores the previous chart *and* values, so the site's customisations come back with it. If the release history itself is unusable, the `/tmp/cassandra-ci-live-values.yaml` from above can be used:
|
||||
```
|
||||
helm upgrade ${RELEASE} jenkins/jenkins --version ${CHART_VERSION} -n ${NS} \
|
||||
-f /tmp/cassandra-ci-live-values.yaml --wait --timeout 15m
|
||||
```
|
||||
|
||||
|
||||
### Configuring Local-only Access
|
||||
|
||||
If you want only local private access to Jenkins, do the following.
|
||||
|
||||
Comment these lines before running `heml upgrade …`
|
||||
Comment these lines before running `helm upgrade …`
|
||||
```
|
||||
# serviceType: LoadBalancer
|
||||
# ingress:
|
||||
# enabled: "true"
|
||||
```
|
||||
Run the heml upgrade and get the password as usual
|
||||
Run the helm upgrade and get the password as usual
|
||||
```
|
||||
helm upgrade --install -f values.yaml cassius jenkins/jenkins --wait
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,11 @@
|
|||
persistence:
|
||||
enabled: true
|
||||
size: "500Gi"
|
||||
# aws needs gp2, gke can be left commented
|
||||
# keep the claim (jobs, credentials, build history) when the release is uninstalled.
|
||||
# deleting it then takes a deliberate `kubectl delete pvc cassius-jenkins`
|
||||
annotations:
|
||||
helm.sh/resource-policy: keep
|
||||
# aws needs gp2, gke can be left commented (add it to your override yaml)
|
||||
#storageClass: "gp2"
|
||||
controller:
|
||||
# To get URL run `kubectl describe svc cassius-jenkins | grep 'LoadBalancer Ingress'`
|
||||
|
|
@ -31,7 +35,7 @@ controller:
|
|||
targetPort: 8080
|
||||
ingress:
|
||||
enabled: "true"
|
||||
# uncomment and set if you have a "ci-cassandra" dns entry for the jenkins controller
|
||||
# if you have a "ci-cassandra" dns entry for the jenkins controller add the following value to your override yaml
|
||||
#hostName: ci-cassandra.<your-domain>
|
||||
customJenkinsLabels:
|
||||
- controller
|
||||
|
|
@ -76,7 +80,7 @@ controller:
|
|||
welcome-message: |
|
||||
jenkins:
|
||||
systemMessage: Welcome to Apache Cassandra
|
||||
# we still need separate jobs because Jenkinsfile can differ (and are read before parameters are applied).
|
||||
# Separate jobs are needed because Jenkinsfiles differ, and are read before parameters are applied.
|
||||
# if a dev branch alters the Jenkinsfile, it will not be picked up by the job – you need to edit the job configuration
|
||||
# see the CAUTION warning in .jenkins/Jenkinsfile
|
||||
# TODO: add new version each release branching
|
||||
|
|
@ -155,20 +159,23 @@ agent:
|
|||
node-selector:
|
||||
cassandra.jenkins.agent: true
|
||||
waitForPodSec: "180"
|
||||
#
|
||||
# Each template below is an opaque string to the helm chart, parsed only by the kubernetes plugin,
|
||||
# so neither the chart nor .jenkins/k8s/jenkins-test.sh can validate what a template asks for.
|
||||
# Two traps live in that gap:
|
||||
# so the chart (nor .jenkins/k8s/jenkins-test.sh) can validate it.
|
||||
#
|
||||
# Two traps then to pay attention to:
|
||||
# - a volume declared under `volumes:` is generated as `volume-N`, and the generated copy wins any
|
||||
# merge with a raw `yaml:` entry of the same name, so a field set only there is silently dropped.
|
||||
# Volumes wanting a field the plugin lacks, a sizeLimit for instance, are declared in `yaml:` alone.
|
||||
# - ephemeral-storage is charged to the pod: every emptyDir, the workspace included, along with the
|
||||
# containers' writable layers and logs, all against the containers' limits summed. Declare no
|
||||
# request and the pod is BestEffort for storage, which the scheduler ignores and the node evicts
|
||||
# first, reporting only `request is 0`. Each template below budgets 80Gi of the ~89Gi that a
|
||||
# 100GiB node allocates, leaving the rest to the node's own image cache and daemonsets.
|
||||
# After any deploy, read back what an agent was actually given:
|
||||
# kubectl get pod -l jenkins/cassius-jenkins-agent -o json \
|
||||
# | jq '.items[0].spec | {volumes, containers: [.containers[] | {name, resources, volumeMounts}]}'
|
||||
# first, reporting only `request is 0`. Each template below budgets 80Gi (of the ~89Gi that a
|
||||
# 100GiB node allocates) leaving the rest to the node's own image cache and daemonsets.
|
||||
#
|
||||
# After any deploy, validate changes like:
|
||||
# kubectl get pod -l jenkins/cassius-jenkins-agent -o json | jq '.items[0].spec | {volumes, containers: [.containers[] | {name, resources, volumeMounts}]}'
|
||||
#
|
||||
podTemplates:
|
||||
agent-dind-small: |
|
||||
- name: agent-dind-small
|
||||
|
|
@ -216,7 +223,7 @@ agent:
|
|||
resourceLimitCpu: 2
|
||||
resourceRequestMemory: 1G
|
||||
resourceLimitMemory: 1G
|
||||
# the workspace emptyDir, charged to the pod, is what this covers
|
||||
# the workspace emptyDir
|
||||
resourceRequestEphemeralStorage: 10Gi
|
||||
resourceLimitEphemeralStorage: 20Gi
|
||||
ttyEnabled: 'true'
|
||||
|
|
@ -243,7 +250,7 @@ agent:
|
|||
resourceLimitCpu: 4
|
||||
resourceRequestMemory: 1G
|
||||
resourceLimitMemory: 2400M
|
||||
# docker's images and containers, in the docker-storage emptyDir, not this container's layer
|
||||
# docker's images and containers, in the docker-storage emptyDir
|
||||
resourceRequestEphemeralStorage: 40Gi
|
||||
resourceLimitEphemeralStorage: 60Gi
|
||||
ttyEnabled: 'true'
|
||||
|
|
@ -267,9 +274,8 @@ agent:
|
|||
- "true"
|
||||
topologyKey: kubernetes.io/hostname
|
||||
# docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge.
|
||||
# 60Gi bounds the images and their containers alone, of the pod's 80Gi: ubuntu-test unpacks
|
||||
# to ~33Gi, and exceeding either evicts the pod naming what it was, which the node's own
|
||||
# threshold does not. fetchDockerImages in Jenkinsfile warns as the node fills.
|
||||
# 60Gi bounds the images and their containers alone, of the pod's 80Gi
|
||||
# fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow.
|
||||
volumes:
|
||||
- name: docker-storage
|
||||
emptyDir:
|
||||
|
|
@ -323,7 +329,7 @@ agent:
|
|||
resourceLimitCpu: 3
|
||||
resourceRequestMemory: 1G
|
||||
resourceLimitMemory: 2400M
|
||||
# the workspace emptyDir, charged to the pod, is what this covers
|
||||
# the workspace emptyDir
|
||||
resourceRequestEphemeralStorage: 10Gi
|
||||
resourceLimitEphemeralStorage: 20Gi
|
||||
ttyEnabled: 'true'
|
||||
|
|
@ -350,7 +356,7 @@ agent:
|
|||
resourceLimitCpu: 4
|
||||
resourceRequestMemory: 3400M
|
||||
resourceLimitMemory: 5G
|
||||
# docker's images and containers, in the docker-storage emptyDir, not this container's layer
|
||||
# docker's images and containers, in the docker-storage emptyDir
|
||||
resourceRequestEphemeralStorage: 40Gi
|
||||
resourceLimitEphemeralStorage: 60Gi
|
||||
ttyEnabled: 'true'
|
||||
|
|
@ -374,9 +380,8 @@ agent:
|
|||
- "true"
|
||||
topologyKey: kubernetes.io/hostname
|
||||
# docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge.
|
||||
# 60Gi bounds the images and their containers alone, of the pod's 80Gi: ubuntu-test unpacks
|
||||
# to ~33Gi, and exceeding either evicts the pod naming what it was, which the node's own
|
||||
# threshold does not. fetchDockerImages in Jenkinsfile warns as the node fills.
|
||||
# 60Gi bounds the images and their containers alone, of the pod's 80Gi
|
||||
# fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow.
|
||||
volumes:
|
||||
- name: docker-storage
|
||||
emptyDir:
|
||||
|
|
@ -430,7 +435,7 @@ agent:
|
|||
resourceLimitCpu: 3
|
||||
resourceRequestMemory: 1G
|
||||
resourceLimitMemory: 2G
|
||||
# the workspace emptyDir, charged to the pod, is what this covers
|
||||
# the workspace emptyDir
|
||||
resourceRequestEphemeralStorage: 10Gi
|
||||
resourceLimitEphemeralStorage: 20Gi
|
||||
ttyEnabled: 'true'
|
||||
|
|
@ -457,7 +462,7 @@ agent:
|
|||
resourceLimitCpu: 7
|
||||
resourceRequestMemory: 16G
|
||||
resourceLimitMemory: 30G
|
||||
# docker's images and containers, in the docker-storage emptyDir, not this container's layer
|
||||
# docker's images and containers, in the docker-storage emptyDir
|
||||
resourceRequestEphemeralStorage: 40Gi
|
||||
resourceLimitEphemeralStorage: 60Gi
|
||||
ttyEnabled: 'true'
|
||||
|
|
@ -481,9 +486,8 @@ agent:
|
|||
- "true"
|
||||
topologyKey: kubernetes.io/hostname
|
||||
# docker's storage, named and mounted here so that the sizeLimit survives the plugin's merge.
|
||||
# 60Gi bounds the images and their containers alone, of the pod's 80Gi: ubuntu-test unpacks
|
||||
# to ~33Gi, and exceeding either evicts the pod naming what it was, which the node's own
|
||||
# threshold does not. fetchDockerImages in Jenkinsfile warns as the node fills.
|
||||
# 60Gi bounds the images and their containers alone, of the pod's 80Gi
|
||||
# fetchDockerImages in Jenkinsfile warns as the node fills as our image sizes grow.
|
||||
volumes:
|
||||
- name: docker-storage
|
||||
emptyDir:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
#!/bin/bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#
|
||||
# Validates the CI declarations under .jenkins/ without deploying anything:
|
||||
# - the Jenkinsfile parses as groovy
|
||||
# - jenkins-deployment.yaml renders through the jenkins helm chart
|
||||
# - the yaml embedded in it (agent pod templates, JCasC config scripts) parses
|
||||
#
|
||||
# Requires: helm, python3 with pyyaml, and either groovy or docker.
|
||||
# Run from anywhere: .jenkins/k8s/jenkins-test.sh
|
||||
|
||||
set -e
|
||||
|
||||
CASSANDRA_DIR="$(cd "$(dirname "$0")/../.." > /dev/null && pwd)"
|
||||
JENKINS_DIR="${CASSANDRA_DIR}/.jenkins"
|
||||
status=0
|
||||
|
||||
command -v helm > /dev/null || { echo "helm must be installed and in the PATH"; exit 1; }
|
||||
command -v python3 > /dev/null || { echo "python3 must be installed and in the PATH"; exit 1; }
|
||||
python3 -c "import yaml" 2> /dev/null || { echo "python3 pyyaml must be installed: pip install pyyaml"; exit 1; }
|
||||
|
||||
echo "== Jenkinsfile groovy syntax"
|
||||
# Phases.CONVERSION parses and builds the AST without resolving the pipeline DSL or @NonCPS,
|
||||
# neither of which exist outside a jenkins controller
|
||||
syntax_check_dir="$(mktemp -d)"
|
||||
syntax_check="${syntax_check_dir}/syntax-check.groovy"
|
||||
# mktemp gives 0700, which the unprivileged user inside the groovy image cannot traverse
|
||||
chmod 755 "${syntax_check_dir}"
|
||||
cat > "${syntax_check}" << 'EOF'
|
||||
import org.codehaus.groovy.control.CompilationUnit
|
||||
import org.codehaus.groovy.control.Phases
|
||||
|
||||
def cu = new CompilationUnit()
|
||||
args.each { cu.addSource(new File(it)) }
|
||||
cu.compile(Phases.CONVERSION)
|
||||
println " ${args.join(', ')} parses"
|
||||
EOF
|
||||
if command -v groovy > /dev/null ; then
|
||||
groovy "${syntax_check}" "${JENKINS_DIR}/Jenkinsfile" || status=1
|
||||
elif command -v docker > /dev/null ; then
|
||||
# absolute paths, the image's working directory is not where the script was mounted
|
||||
docker run --rm -v "${syntax_check_dir}:/check:ro" -v "${JENKINS_DIR}:/jenkins:ro" \
|
||||
groovy:4.0-jdk17 groovy /check/syntax-check.groovy /jenkins/Jenkinsfile || status=1
|
||||
else
|
||||
echo " SKIPPED: neither groovy nor docker found"
|
||||
fi
|
||||
|
||||
echo "== jenkins-deployment.yaml renders through the helm chart"
|
||||
helm repo add jenkins https://charts.jenkins.io > /dev/null
|
||||
helm repo update > /dev/null
|
||||
# --namespace and a release name only so the chart's templates have something to interpolate
|
||||
helm template cassius jenkins/jenkins --namespace default -f "${JENKINS_DIR}/k8s/jenkins-deployment.yaml" > /dev/null \
|
||||
&& echo " jenkins-deployment.yaml renders" || status=1
|
||||
|
||||
echo "== yaml embedded in jenkins-deployment.yaml"
|
||||
python3 - "${JENKINS_DIR}/k8s" << 'EOF' || status=1
|
||||
import sys, yaml
|
||||
from pathlib import Path
|
||||
|
||||
k8s_dir = Path(sys.argv[1])
|
||||
errors = 0
|
||||
|
||||
for path in sorted(k8s_dir.glob("*.yaml")):
|
||||
try:
|
||||
values = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
except yaml.YAMLError as error:
|
||||
print(f" INVALID {path.name}: {error}")
|
||||
errors += 1
|
||||
continue
|
||||
print(f" {path.name} parses")
|
||||
|
||||
# the values in these two maps are themselves yaml documents, and a chart never validates them —
|
||||
# a misindented pod template reaches the kubernetes plugin and silently loses its agents
|
||||
for keys in (("agent", "podTemplates"), ("controller", "JCasC", "configScripts")):
|
||||
embedded = values
|
||||
for key in keys:
|
||||
embedded = embedded.get(key, {}) if isinstance(embedded, dict) else {}
|
||||
for name, document in (embedded or {}).items():
|
||||
location = f"{path.name} {'.'.join(keys)}.{name}"
|
||||
try:
|
||||
parsed = yaml.safe_load(document)
|
||||
except yaml.YAMLError as error:
|
||||
print(f" INVALID {location}: {error}")
|
||||
errors += 1
|
||||
continue
|
||||
print(f" {location} parses")
|
||||
# each pod template may carry a raw kubernetes pod spec in a nested `yaml` key
|
||||
for template in parsed if isinstance(parsed, list) else []:
|
||||
if isinstance(template, dict) and "yaml" in template:
|
||||
try:
|
||||
yaml.safe_load(template["yaml"])
|
||||
except yaml.YAMLError as error:
|
||||
print(f" INVALID {location}.yaml: {error}")
|
||||
errors += 1
|
||||
else:
|
||||
print(f" {location}.yaml parses")
|
||||
|
||||
sys.exit(1 if errors else 0)
|
||||
EOF
|
||||
|
||||
[ 0 -eq ${status} ] && echo "== all .jenkins/ checks passed" || echo "== FAILED"
|
||||
exit ${status}
|
||||
Loading…
Reference in New Issue