diff --git a/.build/ci/ci_parser.py b/.build/ci/ci_parser.py
index 1a6bdf13dc..c91b5f15b9 100755
--- a/.build/ci/ci_parser.py
+++ b/.build/ci/ci_parser.py
@@ -201,8 +201,6 @@ def process_test_cases(suite: JUnitTestSuite, file_name: str, root) -> int:
return test_count
-# TODO: Update this to instead be "create_summary_file" and build the entire summary page, not just append failures to existing
-# This should be trivial to do using JUnitTestSuite.failed, passed, etc methods
def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, output: str) -> None:
"""
Will create a table with all failed tests in it organized by sorted suite name.
@@ -218,6 +216,8 @@ def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, outpu
with open(output, 'r') as file:
soup = BeautifulSoup(file, 'html.parser')
+ failures_list_tag = soup.new_tag("div")
+ failures_list_tag.string = '
[Test Failures]
'
failures_tag = soup.new_tag("div")
failures_tag.string = '
[Test Failure Details]
'
suites_tag = soup.new_tag("div")
@@ -250,6 +250,7 @@ def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, outpu
failures_builder.label_columns(['Class', 'Method', 'Output', 'Duration'], ["width: 15%; text-align: left;", "width: 15%; text-align: left;", "width: 60%; text-align: left;", "width: 10%; text-align: right;"])
for test in suite.get_tests(JUnitTestStatus.FAILURE):
failures_builder.add_row(test.row_data())
+ failures_list_tag.append(BeautifulSoup(failures_builder.build_list(), 'html.parser'))
failures_tag.append(BeautifulSoup(failures_builder.build_table(), 'html.parser'))
total_failure_count += failure_count
if total_failure_count > 200:
@@ -269,6 +270,7 @@ def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, outpu
"""
soup.body.append(totals_tag)
+ soup.body.append(failures_list_tag)
soup.body.append(failures_tag)
suites_tag.append(BeautifulSoup(suites_builder.build_table(), 'html.parser'))
soup.body.append(suites_tag)
diff --git a/.build/ci/junit_helpers.py b/.build/ci/junit_helpers.py
index d7d3702e99..75ffa08534 100644
--- a/.build/ci/junit_helpers.py
+++ b/.build/ci/junit_helpers.py
@@ -101,6 +101,14 @@ class JUnitResultBuilder:
''')
+ self._list_template = Template('''
+
+ {% for row in rows %}
+ {{ row[0] }} {{ row[1] }}
+ {% endfor %}
+
+ ''')
+
@staticmethod
def add_style_tags(soup: BeautifulSoup) -> None:
"""
@@ -130,6 +138,9 @@ class JUnitResultBuilder:
raise AssertionError(f'Got invalid number of columns on add_row: {len(row)}. Expected: 4.')
self._rows.append(row)
+ def build_list(self) -> str:
+ return self._list_template.render(rows=self._rows)
+
def build_table(self) -> str:
return self._template.render(header=f'{self._name}', labels=self._labels, column_styles=self._column_styles, rows=self._rows)
diff --git a/.build/docker/_build-debian.sh b/.build/docker/_build-debian.sh
index be9dbca693..7d92d6eb07 100755
--- a/.build/docker/_build-debian.sh
+++ b/.build/docker/_build-debian.sh
@@ -96,7 +96,7 @@ else
# if CASSANDRA_VERSION is -alphaN, -betaN, -rcN, it fails on the '-' char; replace with '~'
CASSANDRA_VERSION=${buildxml_version/-/\~}
dt=`date +"%Y%m%d"`
- ref=`git rev-parse --short HEAD`
+ ref=`git rev-parse --short HEAD || grep -q GitSHA src/resources/org/apache/cassandra/config/version.properties && grep GitSHA src/resources/org/apache/cassandra/config/version.properties | cut -d"=" -f2 || echo unknown`
CASSANDRA_REVISION="${dt}git${ref}"
dch -D unstable -v "${CASSANDRA_VERSION}-${CASSANDRA_REVISION}" --package "cassandra" "building ${CASSANDRA_VERSION}-${CASSANDRA_REVISION}"
fi
@@ -120,11 +120,12 @@ fi
# build package
dpkg-buildpackage -rfakeroot -uc -us -tc --source-option=--tar-ignore=.git
+set +e
# Move created artifacts to dist dir mapped to docker host directory (must have proper permissions)
mv ../cassandra[-_]*${CASSANDRA_VERSION}* "${DIST_DIR}"
# clean build deps
rm -f cassandra-build-deps_*
# restore debian/changelog
-git restore debian/changelog
+git restore debian/changelog || true
popd >/dev/null
\ No newline at end of file
diff --git a/.build/docker/_build-redhat.sh b/.build/docker/_build-redhat.sh
index 37ca50799a..6d76b30bd6 100755
--- a/.build/docker/_build-redhat.sh
+++ b/.build/docker/_build-redhat.sh
@@ -104,7 +104,7 @@ else
# from the branch name. In this case, fall back to version specified in build.xml.
CASSANDRA_VERSION="${buildxml_version}"
dt=`date +"%Y%m%d"`
- ref=`git rev-parse --short HEAD`
+ ref=`git rev-parse --short HEAD || grep -q GitSHA src/resources/org/apache/cassandra/config/version.properties && grep GitSHA src/resources/org/apache/cassandra/config/version.properties | cut -d"=" -f2 || echo unknown`
CASSANDRA_REVISION="${dt}git${ref}"
fi
diff --git a/.build/docker/_docker_init_tests.sh b/.build/docker/_docker_init_tests.sh
index 1efbd87526..cd759c5cd9 100755
--- a/.build/docker/_docker_init_tests.sh
+++ b/.build/docker/_docker_init_tests.sh
@@ -44,4 +44,5 @@ fi
# these happen when the image hasn't pre-downloaded all the ccm versions used in tests
rm -rf /tmp/ccm-*.tar.gz
+set -x
exit ${status}
\ No newline at end of file
diff --git a/.build/docker/_docker_run.sh b/.build/docker/_docker_run.sh
index 5d93563484..6aafc74157 100755
--- a/.build/docker/_docker_run.sh
+++ b/.build/docker/_docker_run.sh
@@ -95,15 +95,20 @@ image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}"
# Look for existing docker image, otherwise build
if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
- # try docker login to increase dockerhub rate limits
- timeout -k 5 5 docker login >/dev/null
+ echo "Build image not found locally, pulling image ${image_name}..."
if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
# Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
+ echo "Building docker image..."
until docker build -t ${image_name} -f docker/${dockerfile} . ; do
echo "docker build failed… trying again in 10s… "
sleep 10
done
+ echo "Docker image ${image_name} has been built"
+ else
+ echo "Successfully pulled build image."
fi
+else
+ echo "Found build image locally."
fi
# Run build script through docker
diff --git a/.build/docker/centos7-build.docker b/.build/docker/centos7-build.docker
index 9f9782ba80..901e77985d 100644
--- a/.build/docker/centos7-build.docker
+++ b/.build/docker/centos7-build.docker
@@ -23,7 +23,7 @@ ENV BUILD_HOME=/home/build
ENV RPM_BUILD_DIR=$BUILD_HOME/rpmbuild
ENV DIST_DIR=/dist
ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
-ENV ANT_VERSION=1.10.12
+ENV ANT_VERSION=1.10.14
ARG UID_ARG=1000
ARG GID_ARG=1000
diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh
index 17e2682a31..783f914c72 100755
--- a/.build/docker/run-tests.sh
+++ b/.build/docker/run-tests.sh
@@ -30,6 +30,12 @@ if [ "$#" -lt 1 ] || [ "$#" -gt 3 ] || [ "$1" == "-h" ]; then
exit 1
fi
+error() {
+ echo >&2 $2;
+ set -x
+ exit $1
+}
+
# variables, with defaults
[ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname "$0")/../..)"
[ "x${cassandra_dtest_dir}" != "x" ] || cassandra_dtest_dir="${cassandra_dir}/../cassandra-dtest"
@@ -39,12 +45,12 @@ fi
[ -d "${m2_dir}" ] || { mkdir -p "${m2_dir}" ; }
# pre-conditions
-command -v docker >/dev/null 2>&1 || { echo >&2 "docker needs to be installed"; exit 1; }
-command -v bc >/dev/null 2>&1 || { echo >&2 "bc needs to be installed"; exit 1; }
-command -v timeout >/dev/null 2>&1 || { echo >&2 "timeout needs to be installed"; exit 1; }
-(docker info >/dev/null 2>&1) || { echo >&2 "docker needs to running"; exit 1; }
-[ -f "${cassandra_dir}/build.xml" ] || { echo >&2 "${cassandra_dir}/build.xml must exist"; exit 1; }
-[ -f "${cassandra_dir}/.build/run-tests.sh" ] || { echo >&2 "${cassandra_dir}/.build/run-tests.sh must exist"; exit 1; }
+command -v docker >/dev/null 2>&1 || { error 1 "docker needs to be installed"; }
+command -v bc >/dev/null 2>&1 || { error 1 "bc needs to be installed"; }
+command -v timeout >/dev/null 2>&1 || { error 1 "timeout needs to be installed"; }
+(docker info >/dev/null 2>&1) || { error 1 "docker needs to running"; }
+[ -f "${cassandra_dir}/build.xml" ] || { error 1 "${cassandra_dir}/build.xml must exist"; }
+[ -f "${cassandra_dir}/.build/run-tests.sh" ] || { error 1 "${cassandra_dir}/.build/run-tests.sh must exist"; }
# arguments
target=$1
@@ -63,8 +69,7 @@ fi
regx_java_version="(${java_version_supported//,/|})"
if [[ ! "${java_version}" =~ $regx_java_version ]]; then
- echo "Error: Java version is not in ${java_version_supported}, it is set to ${java_version}"
- exit 1
+ error 1 "Error: Java version is not in ${java_version_supported}, it is set to ${java_version}"
fi
# allow python version override, otherwise default to current python version or 3.8
@@ -88,14 +93,10 @@ docker_mounts="${docker_mounts} -v "${build_dir}/tmp":/home/cassandra/cassandra/
# Look for existing docker image, otherwise build
if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
- echo "Build image not found locally"
- # try docker login to increase dockerhub rate limits
- echo "Attempting 'docker login' to increase dockerhub rate limits"
- timeout -k 5 5 docker login >/dev/null 2>/dev/null
- echo "Pulling build image..."
+ echo "Build image not found locally, pulling image ${image_name}..."
if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
# Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
- echo "Building docker image ${image_name}..."
+ echo "Building docker image..."
until docker build -t ${image_name} -f docker/${dockerfile} . ; do
echo "docker build failed… trying again in 10s… "
sleep 10
@@ -103,9 +104,9 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
echo "Docker image ${image_name} has been built"
else
echo "Successfully pulled build image."
- fi
+ fi
else
- echo "Found build image locally."
+ echo "Found build image locally."
fi
pushd ${cassandra_dir} >/dev/null
@@ -119,47 +120,47 @@ if [[ ! -z ${JENKINS_URL+x} ]] && [[ ! -z ${NODE_NAME+x} ]] ; then
fi
# find host's available cores and mem
-cores=$(docker run --rm alpine nproc --all) || { echo >&2 "Unable to check available CPU cores"; exit 1; }
+cores=$(docker run --rm alpine:3.19.1 nproc --all) || { error 1 "Unable to check available CPU cores"; }
case $(uname) in
"Linux")
- mem=$(docker run --rm alpine free -b | grep Mem: | awk '{print $2}') || { echo >&2 "Unable to check available memory"; exit 1; }
+ mem=$(docker run --rm alpine:3.19.1 free -b | grep Mem: | awk '{print $2}') || { error 1 "Unable to check available memory"; }
;;
"Darwin")
- mem=$(sysctl -n hw.memsize) || { echo >&2 "Unable to check available memory"; exit 1; }
+ mem=$(sysctl -n hw.memsize) || { error 1 "Unable to check available memory"; }
;;
*)
- echo >&2 "Unsupported operating system, expected Linux or Darwin"
- exit 1
+ error 1 "Unsupported operating system, expected Linux or Darwin"
esac
# figure out resource limits, scripts, and mounts for the test type
+docker_flags="-m 5g --memory-swap 5g"
case ${target} in
"build_dtest_jars")
;;
"stress-test" | "fqltool-test" )
- [[ ${mem} -gt $((1 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 1g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
+ [[ ${mem} -gt $((1 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 1g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
;;
# test-burn doesn't have enough tests in it to split beyond 8, and burn and long we want a bit more resources anyway
"microbench" | "test-burn" | "long-test" | "cqlsh-test" )
- [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
+ [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
;;
- "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" )
- [ -f "${cassandra_dtest_dir}/dtest.py" ] || { echo >&2 "${cassandra_dtest_dir}/dtest.py must exist"; exit 1; }
- [[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
+ "simulator-dtest")
+ [[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
+ docker_flags="-m 15g --memory-swap 15g"
+ ;;
+ "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large")
+ [ -f "${cassandra_dtest_dir}/dtest.py" ] || { error 1 "${cassandra_dtest_dir}/dtest.py not found. please specify 'cassandra_dtest_dir' to point to the local cassandra-dtest source"; }
test_script="run-python-dtests.sh"
docker_mounts="${docker_mounts} -v ${cassandra_dtest_dir}:/home/cassandra/cassandra-dtest"
- # check that ${cassandra_dtest_dir} is valid
- [ -f "${cassandra_dtest_dir}/dtest.py" ] || { echo >&2 "${cassandra_dtest_dir}/dtest.py not found. please specify 'cassandra_dtest_dir' to point to the local cassandra-dtest source"; exit 1; }
+ [[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
+ docker_flags="-m 15g --memory-swap 15g"
;;
- "test"| "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode" | "simulator-dtest")
- [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
- max_docker_runs_by_cores=$( echo "sqrt( ${cores} / ${jenkins_executors} )" | bc )
- max_docker_runs_by_mem=$(( ${mem} / ( 5 * 1024 * 1024 * 1024 * ${jenkins_executors} ) ))
+ "test"| "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
+ [[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
;;
*)
- echo "unrecognized test type \"${target}\""
- exit 1
+ error 1 "unrecognized test type \"${target}\""
;;
esac
@@ -171,12 +172,8 @@ if (( $(echo "${docker_cpus} > ${docker_cpus_limit}" |bc -l) )) ; then
fi
# hack: long-test does not handle limited CPUs
-if [ "${target}" == "long-test" ] ; then
- docker_flags="-m 5g --memory-swap 5g"
-elif [[ "${target}" =~ dtest* ]] ; then
- docker_flags="--cpus=${docker_cpus} -m 15g --memory-swap 15g"
-else
- docker_flags="--cpus=${docker_cpus} -m 5g --memory-swap 5g"
+if [ "${target}" != "long-test" ] ; then
+ docker_flags="--cpus=${docker_cpus} ${docker_flags}"
fi
docker_flags="${docker_flags} -d --rm"
@@ -203,7 +200,7 @@ esac
# cython can be used for cqlsh-test
if [ "$cython" == "yes" ]; then
- [ "${target}" == "cqlsh-test" ] || { echo "cython is only supported for cqlsh-test"; exit 1; }
+ [ "${target}" == "cqlsh-test" ] || { error 1 "cython is only supported for cqlsh-test"; }
ANT_OPTS="${ANT_OPTS}_cython"
else
cython="no"
@@ -268,4 +265,5 @@ xz -f ${logfile} 2>/dev/null
popd >/dev/null
popd >/dev/null
+set -x
exit ${status}
diff --git a/.build/docker/ubuntu2004_test.docker b/.build/docker/ubuntu2004_test.docker
index 3e15b114d7..a27bd55412 100644
--- a/.build/docker/ubuntu2004_test.docker
+++ b/.build/docker/ubuntu2004_test.docker
@@ -85,6 +85,7 @@ RUN echo "cassandra-tmp ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/build
RUN chmod 0440 /etc/sudoers.d/build
# switch to the cassandra user
+RUN mkdir -p ${BUILD_HOME} && chmod a+rwx ${BUILD_HOME}
USER cassandra-tmp
ENV HOME ${BUILD_HOME}
WORKDIR ${BUILD_HOME}
diff --git a/.build/run-tests.sh b/.build/run-tests.sh
index 21fbf77c9b..e36cd77d6d 100755
--- a/.build/run-tests.sh
+++ b/.build/run-tests.sh
@@ -172,7 +172,7 @@ _main() {
fi
# check project is already built. no cleaning is done, so jenkins unstash works, beware.
- [[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR})"; exit 1; }
+ [[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR} | xargs)"; exit 1; }
# check if dist artifacts exist, this breaks the dtests
[[ -d "${DIST_DIR}/dist" ]] && { echo "tests don't work when build/dist ("${DIST_DIR}/dist") exists (from \`ant artifacts\`)"; exit 1; }
diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile
index 1bb2fc0e11..8f497aff0b 100644
--- a/.jenkins/Jenkinsfile
+++ b/.jenkins/Jenkinsfile
@@ -48,6 +48,8 @@
// will be ignored when run on other environments.
// Note there are also differences when CI is being run pre- or post-commit.
//
+// CAUTION! When running CI with changes in this file, ensure the "Pipeline script from SCM" scm details match
+// the brances being tested. These details don't honour the per-build repository and branch parameterisation.
//
// Validate/lint this file using the following command
// `curl -X POST -F "jenkinsfile=<.jenkins/Jenkinsfile" https://ci-cassandra.apache.org/pipeline-model-converter/validate`
@@ -56,18 +58,22 @@
pipeline {
agent { label 'cassandra-small' }
+ options {
+ // must have: avoids agents waste in idle time on controller bottleneck
+ durabilityHint('PERFORMANCE_OPTIMIZED')
+ }
parameters {
- string(name: 'repository', defaultValue: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository')
- string(name: 'branch', defaultValue: env.BRANCH_NAME, description: 'Branch')
+ string(name: 'repository', defaultValue: params.repository ?: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository')
+ string(name: 'branch', defaultValue: params.branch ?: scm.userRemoteConfigs[0].refspec, description: 'Branch')
- choice(name: 'profile', choices: pipelineProfiles().keySet() as List, description: 'Pick a pipeline profile.')
- string(name: 'profile_custom_regexp', defaultValue: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*')
+ choice(name: 'profile', choices: pipelineProfileNames(params.profile ?: ''), description: 'Pick a pipeline profile.')
+ string(name: 'profile_custom_regexp', defaultValue: params.profile_custom_regexp ?: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*')
choice(name: 'architecture', choices: archsSupported() + "all", description: 'Pick architecture. The ARM64 is disabled by default at the moment.')
- string(name: 'jdk', defaultValue: "", description: 'Restrict JDK versions. (e.g. "11", "17", etc)')
+ string(name: 'jdk', defaultValue: params.jdk ?: '', description: 'Restrict JDK versions. (e.g. "11", "17", etc)')
- string(name: 'dtest_repository', defaultValue: 'https://github.com/apache/cassandra-dtest' ,description: 'Cassandra DTest Repository')
- string(name: 'dtest_branch', defaultValue: 'trunk', description: 'DTest Branch')
+ string(name: 'dtest_repository', defaultValue: params.dtest_repository ?: 'https://github.com/apache/cassandra-dtest', description: 'Cassandra DTest Repository')
+ string(name: 'dtest_branch', defaultValue: params.dtest_branch ?: 'trunk', description: 'DTest Branch')
}
stages {
stage('jar') {
@@ -125,6 +131,13 @@ def pipelineProfiles() {
]
}
+def pipelineProfileNames(putFirst) {
+ set = pipelineProfiles().keySet() as List
+ set = set - putFirst
+ set.add(0, putFirst)
+ return set
+}
+
def tasks() {
// Steps config
def buildSteps = [
@@ -152,9 +165,9 @@ def tasks() {
'long-test': [splits: 8],
'test-oa': [splits: 8],
'test-system-keyspace-directory': [splits: 8],
- 'jvm-dtest': [splits: 8],
+ 'jvm-dtest': [splits: 12],
'jvm-dtest-upgrade': [splits: 8],
- 'simulator-dtest': [splits: 1],
+ 'simulator-dtest': [splits: 1, size: 'large'],
'dtest': [splits: 64, size: 'large'],
'dtest-novnode': [splits: 64, size: 'large'],
'dtest-latest': [splits: 64, size: 'large'],
@@ -221,7 +234,7 @@ def tasks() {
@NonCPS
def List getMatrixAxes(Map matrix_axes) {
- List axes = []
+ def List axes = []
matrix_axes.each { axis, values ->
List axisList = []
values.each { value ->
@@ -233,10 +246,10 @@ def List getMatrixAxes(Map matrix_axes) {
}
def getStepName(cell, command) {
- arch = "amd64" == cell.arch ? "" : " ${cell.arch}"
- python = "cqlsh-test" != cell.step ? "" : " python${cell.python}"
- cython = "no" == cell.cython ? "" : " cython"
- split = command.splits > 1 ? " ${cell.split}/${command.splits}" : ""
+ def arch = "amd64" == cell.arch ? "" : " ${cell.arch}"
+ def python = "cqlsh-test" != cell.step ? "" : " python${cell.python}"
+ def cython = "no" == cell.cython ? "" : " cython"
+ def split = command.splits > 1 ? " ${cell.split}/${command.splits}" : ""
return "${cell.step}${arch} jdk${cell.jdk}${python}${cython}${split}"
}
@@ -286,7 +299,7 @@ def isCanonical() {
}
def isStageEnabled(stage) {
- return "jar" == stage || pipelineProfiles()[params.profile].contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp)
+ return "jar" == stage || pipelineProfiles()[params.profile]?.contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp)
}
def isArchEnabled(arch) {
@@ -311,26 +324,24 @@ def build(command, cell) {
nodeExclusion = "&&!${NODE_NAME}"
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
- cleanAgent(cell.step)
- cleanWs()
fetchSource(cell.step, cell.arch, cell.jdk)
sh """
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
- """
+ """
+ fetchDockerImages(['almalinux-build', 'bullseye-build', 'centos7-build'])
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
- def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log"
+ def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
- status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee build/${logfile}", returnStatus: true
+ def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
dir("build") {
- sh "xz -f *${logfile}"
- archiveArtifacts artifacts: "${logfile}.xz", fingerprint: true
- copyToNightlies("${logfile}.xz", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
+ archiveArtifacts artifacts: "${logfile}", fingerprint: true
+ copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
}
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
if ("jar" == cell.step) { // TODO only stash the project built files. all dependency libraries are restored from the local maven repo using `ant resolver-dist-lib`
- stash name: "${cell.arch}_${cell.jdk}", useDefaultExcludes: false //, includes: '**/*.jar' //, includes: "*.jar,classes/**,test/classes/**,tools/**"
+ stash name: "${cell.arch}_${cell.jdk}"
}
dir("build") {
copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
@@ -353,11 +364,10 @@ def test(command, cell) {
nodeExclusion = "&&!${NODE_NAME}"
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
- cleanAgent(cell.step)
- cleanWs()
fetchSource(cell.step, cell.arch, cell.jdk)
+ fetchDockerImages(['ubuntu2004_test'])
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
- def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log"
+ def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
// pipe to tee needs pipefail
def script_vars = "#!/bin/bash \n set -o pipefail ; "
script_vars = "${script_vars} python_version=\'${cell.python}\'"
@@ -367,11 +377,10 @@ def test(command, cell) {
}
script_vars = fetchDTestsSource(command, script_vars)
buildJVMDTestJars(cell, script_vars, logfile)
- status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee -a build/${logfile}", returnStatus: true
+ def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
dir("build") {
- sh "xz -f ${logfile}"
- archiveArtifacts artifacts: "${logfile}.xz", fingerprint: true
- copyToNightlies("${logfile}.xz", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
+ archiveArtifacts artifacts: "${logfile}", fingerprint: true
+ copyToNightlies("${logfile}", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
}
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
dir("build") {
@@ -394,17 +403,18 @@ def test(command, cell) {
}
def fetchSource(stage, arch, jdk) {
- if ("jar" == stage) {
- checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
- sh "mkdir -p build/stage-logs"
- } else {
- unstash name: "${arch}_${jdk}"
- }
+ cleanAgent(stage)
+ if ("jar" == stage) {
+ checkout changelog: false, scm: scmGit(branches: [[name: params.branch ?: 'trunk']], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
+ sh "mkdir -p build/stage-logs"
+ } else {
+ unstash name: "${arch}_${jdk}"
+ }
}
def fetchDTestsSource(command, script_vars) {
if (command.containsKey('python-dtest')) {
- checkout changelog: false, poll: false, scm: scmGit(branches: [[name: params.dtest_branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true), [$class: 'RelativeTargetDirectory', relativeTargetDir: "${WORKSPACE}/build/cassandra-dtest"]], userRemoteConfigs: [[url: params.dtest_repository]])
+ checkout changelog: false, poll: false, scm: scmGit(branches: [[name: params.dtest_branch ?: 'trunk']], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true), [$class: 'RelativeTargetDirectory', relativeTargetDir: "${WORKSPACE}/build/cassandra-dtest"]], userRemoteConfigs: [[url: params.dtest_repository]])
sh "test -f build/cassandra-dtest/requirements.txt || { echo 'Invalid cassandra-dtest fork/branch'; exit 1; }"
return "${script_vars} cassandra_dtest_dir='${WORKSPACE}/build/cassandra-dtest'"
}
@@ -416,12 +426,29 @@ def buildJVMDTestJars(cell, script_vars, logfile) {
try {
unstash name: "jvm_dtests_${cell.arch}_${cell.jdk}"
} catch (error) {
- sh label: "RUNNING build_dtest_jars...", script: "${script_vars} .build/docker/run-tests.sh build_dtest_jars ${cell.jdk} 2>&1 | tee build/${logfile}"
+ sh label: "RUNNING build_dtest_jars...", script: "${script_vars} .build/docker/run-tests.sh build_dtest_jars ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )"
stash name: "jvm_dtests_${cell.arch}_${cell.jdk}", includes: '**/dtest*.jar'
}
}
}
+def fetchDockerImages(dockerfiles) {
+ // prefetch, from apache jfrog, reduces risking dockerhub pull rate limits
+ // also prefetch alpine:latest as its used as a utility in the scripts
+ def dockerfilesVar = dockerfiles.join(' ')
+ sh """#!/bin/bash
+ for dockerfile in ${dockerfilesVar} ; do
+ image_tag="\$(md5sum .build/docker/\${dockerfile}.docker | cut -d' ' -f1)"
+ image_name="apache/cassandra-\${dockerfile}:\${image_tag}"
+ if ! ( [[ "" != "\$(docker images -q \${image_name} 2>/dev/null)" ]] ) ; then
+ docker pull -q apache.jfrog.io/cassan-docker/\${image_name} &
+ fi
+ done
+ docker pull -q apache.jfrog.io/cassan-docker/alpine:3.19.1 &
+ wait
+ """
+}
+
def getNodeLabel(command, cell) {
echo "using node label: cassandra-${cell.arch}-${command.size}"
return "cassandra-${cell.arch}-${command.size}"
@@ -434,13 +461,13 @@ def copyToNightlies(sourceFiles, remoteDirectory='') {
retry(9) {
if (attempt > 1) { sleep(60 * attempt) }
sshPublisher(
- continueOnError: true, failOnError: false,
- publishers: [
- sshPublisherDesc(
- configName: "Nightlies",
- transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ]
- )
- ])
+ continueOnError: true, failOnError: false,
+ publishers: [
+ sshPublisherDesc(
+ configName: "Nightlies",
+ transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ]
+ )
+ ])
}
echo "archived to https://nightlies.apache.org/${remotePath}"
}
@@ -449,13 +476,32 @@ def copyToNightlies(sourceFiles, remoteDirectory='') {
def cleanAgent(job_name) {
sh "hostname"
if (isCanonical()) {
- def maxJobHours = 12
- echo "Cleaning project, and pruning docker for '${job_name}' on ${NODE_NAME}…" ;
- sh """
- git clean -qxdff -e build/test/jmh-result.json || true;
- if pgrep -xa docker || pgrep -af "build/docker" || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --all --force --filter "until=${maxJobHours}h" || true ; else docker system prune --force --volumes || true ; fi;
- """
+ def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/"
+ cleanAgentDocker(job_name, agentScriptsUrl)
+ logAgentInfo(job_name, agentScriptsUrl)
}
+ cleanWs()
+}
+
+def cleanAgentDocker(job_name, agentScriptsUrl) {
+ // we don't expect any build to have been running for longer than maxBuildHours
+ def maxBuildHours = 12
+ echo "Pruning docker for '${job_name}' on ${NODE_NAME}…" ;
+ sh """#!/bin/bash
+ set +e
+ wget -q ${agentScriptsUrl}/docker_image_pruner.py
+ wget -q ${agentScriptsUrl}/docker_agent_cleaner.sh
+ bash docker_agent_cleaner.sh ${maxBuildHours}
+ """
+}
+
+def logAgentInfo(job_name, agentScriptsUrl) {
+ sh """#!/bin/bash
+ set +e -o pipefail
+ wget -q ${agentScriptsUrl}/agent_report.sh
+ bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt
+ """
+ copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/")
}
/////////////////////////////////////////
@@ -463,18 +509,29 @@ def cleanAgent(job_name) {
/////////////////////////////////////////
def generateTestReports() {
- // built-in on ci-cassandra will be much faster (local transfer for copyArtifacts and archiveArtifacts)
- def nodeName = isCanonical() ? "built-in" : "cassandra-medium"
- node(nodeName) {
- cleanWs()
+ node("cassandra-medium") {
+ cleanAgent("generateTestReports")
checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
- copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
+ def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_generateTestReports.log.xz"
+ sh "mkdir -p build/stage-logs"
+ def teeSuffix = "2>&1 | tee >( xz -c > build/${logfile} )"
+ def script_vars = "#!/bin/bash -x \n "
+ if (isCanonical()) {
+ // copyArtifacts takes >4hrs, hack with manual download
+ sh """${script_vars}
+ ( mkdir -p build/test
+ wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip
+ unzip -x -d build/test -q output.zip ) ${teeSuffix}
+ """
+ } else {
+ copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
+ }
if (fileExists('build/test/output')) {
// merge splits for each target's test report, other axes are kept separate
// TODO parallelised for loop
// TODO results_details.tar.xz needs to include all logs for failed tests
- sh """
- find build/test/output -name *.xml.xz -exec sh -c 'xz -f --decompress {} &' ';' ; wait
+ sh """${script_vars} (
+ find build/test/output -name *.xml.xz -exec sh -c 'xz -f --decompress {} &' ';' ; wait
for target in \$(ls build/test/output/) ; do
if test -d build/test/output/\${target} ; then
@@ -488,12 +545,13 @@ def generateTestReports() {
.build/docker/_docker_run.sh bullseye-build.docker ci/generate-ci-summary.sh || echo "failed generate-ci-summary.sh"
- tar -cf build/results_details.tar -C build/test/ reports && xz -9f build/results_details.tar
+ tar -cf build/results_details.tar -C build/test/ reports
+ xz -8f build/results_details.tar ) ${teeSuffix}
"""
dir('build/') {
- archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz", fingerprint: true
- copyToNightlies('results_details.tar.xz')
+ archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true
+ copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile}')
}
}
}
@@ -502,6 +560,7 @@ def generateTestReports() {
def sendNotifications() {
if (isPostCommit() && isCanonical()) {
// the following is expected only to work on ci-cassandra.apache.org
+ def changes = '?'
try {
script {
changes = formatChangeLogChanges(currentBuild.changeSets)
diff --git a/build.xml b/build.xml
index adba60d9a7..479dfac767 100644
--- a/build.xml
+++ b/build.xml
@@ -1851,7 +1851,7 @@
-
+