mirror of https://github.com/apache/cassandra
Honour parameter defaults between builds in Jenkinsfile
Also – increase splits for jvm-dtests, container size and docker limits for simulator-dtest – print exit status in .build/docker/run-tests.sh and .build/docker/_docker_init_tests.sh – add a summary list of test failures in ci_summary.html – in ubuntu2004_test.docker make sure /home/cassandra exists and has correct perms (from Marcuse) – when on ci-cassandra, replace use of copyArtifacts in Jenkinsfile generateTestReports() with manual wget of test files (copyArtifacts is notoriously slow, >4hrs in this case) – copy ci_summary.html and results_details.tar.xz to nightlies – capture generateTestReports logs – stream xz where possible, instead of xz compressing afterwards – fix ant version in centos7-build.docker – remove docker login (was meangingless, if credentials exist then docker is logged in) – prefetch docker images from jfrog (to reduce dockerhub pull rate limits), relates to CASSANDRA-18931 – use scripts from cassandra-builds to clean and report on agents, relates to CASSANDRA-18130 – use groovy elvis operators to avoid NPEs/failures on blank param values – parallel jfrog pulls and cache it in jfrog too – pin alpine docker image version – don't stash .git (was causing AccessDeniedException on some unstash), needed fix so deb/rpm packaging worked on not-git work directories – durabilityHint needs PERFORMANCE_OPTIMIZED, controller becomes scale bottleneck and breaks without it patch by Mick Semb Wever; reviewed by Brandon Williams for CASSANDRA-19558
This commit is contained in:
Mick Semb Wever
parent
fda12b25d2
commit
3c85def5cc
|
|
@ -201,8 +201,6 @@ def process_test_cases(suite: JUnitTestSuite, file_name: str, root) -> int:
|
|||
return test_count
|
||||
|
||||
|
||||
# TODO: Update this to instead be "create_summary_file" and build the entire summary page, not just append failures to existing
|
||||
# This should be trivial to do using JUnitTestSuite.failed, passed, etc methods
|
||||
def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, output: str) -> None:
|
||||
"""
|
||||
Will create a table with all failed tests in it organized by sorted suite name.
|
||||
|
|
@ -218,6 +216,8 @@ def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, outpu
|
|||
with open(output, 'r') as file:
|
||||
soup = BeautifulSoup(file, 'html.parser')
|
||||
|
||||
failures_list_tag = soup.new_tag("div")
|
||||
failures_list_tag.string = '<br/><br/>[Test Failures]<br/><br/>'
|
||||
failures_tag = soup.new_tag("div")
|
||||
failures_tag.string = '<br/><br/>[Test Failure Details]<br/><br/>'
|
||||
suites_tag = soup.new_tag("div")
|
||||
|
|
@ -250,6 +250,7 @@ def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, outpu
|
|||
failures_builder.label_columns(['Class', 'Method', 'Output', 'Duration'], ["width: 15%; text-align: left;", "width: 15%; text-align: left;", "width: 60%; text-align: left;", "width: 10%; text-align: right;"])
|
||||
for test in suite.get_tests(JUnitTestStatus.FAILURE):
|
||||
failures_builder.add_row(test.row_data())
|
||||
failures_list_tag.append(BeautifulSoup(failures_builder.build_list(), 'html.parser'))
|
||||
failures_tag.append(BeautifulSoup(failures_builder.build_table(), 'html.parser'))
|
||||
total_failure_count += failure_count
|
||||
if total_failure_count > 200:
|
||||
|
|
@ -269,6 +270,7 @@ def create_summary_file(test_suites: Dict[str, JUnitTestSuite], xml_files, outpu
|
|||
"""
|
||||
|
||||
soup.body.append(totals_tag)
|
||||
soup.body.append(failures_list_tag)
|
||||
soup.body.append(failures_tag)
|
||||
suites_tag.append(BeautifulSoup(suites_builder.build_table(), 'html.parser'))
|
||||
soup.body.append(suites_tag)
|
||||
|
|
|
|||
|
|
@ -101,6 +101,14 @@ class JUnitResultBuilder:
|
|||
</table>
|
||||
''')
|
||||
|
||||
self._list_template = Template('''
|
||||
<div>
|
||||
{% for row in rows %}
|
||||
{{ row[0] }} {{ row[1] }}<br/>
|
||||
{% endfor %}
|
||||
</div>
|
||||
''')
|
||||
|
||||
@staticmethod
|
||||
def add_style_tags(soup: BeautifulSoup) -> None:
|
||||
"""
|
||||
|
|
@ -130,6 +138,9 @@ class JUnitResultBuilder:
|
|||
raise AssertionError(f'Got invalid number of columns on add_row: {len(row)}. Expected: 4.')
|
||||
self._rows.append(row)
|
||||
|
||||
def build_list(self) -> str:
|
||||
return self._list_template.render(rows=self._rows)
|
||||
|
||||
def build_table(self) -> str:
|
||||
return self._template.render(header=f'{self._name}', labels=self._labels, column_styles=self._column_styles, rows=self._rows)
|
||||
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ else
|
|||
# if CASSANDRA_VERSION is -alphaN, -betaN, -rcN, it fails on the '-' char; replace with '~'
|
||||
CASSANDRA_VERSION=${buildxml_version/-/\~}
|
||||
dt=`date +"%Y%m%d"`
|
||||
ref=`git rev-parse --short HEAD`
|
||||
ref=`git rev-parse --short HEAD || grep -q GitSHA src/resources/org/apache/cassandra/config/version.properties && grep GitSHA src/resources/org/apache/cassandra/config/version.properties | cut -d"=" -f2 || echo unknown`
|
||||
CASSANDRA_REVISION="${dt}git${ref}"
|
||||
dch -D unstable -v "${CASSANDRA_VERSION}-${CASSANDRA_REVISION}" --package "cassandra" "building ${CASSANDRA_VERSION}-${CASSANDRA_REVISION}"
|
||||
fi
|
||||
|
|
@ -120,11 +120,12 @@ fi
|
|||
# build package
|
||||
dpkg-buildpackage -rfakeroot -uc -us -tc --source-option=--tar-ignore=.git
|
||||
|
||||
set +e
|
||||
# Move created artifacts to dist dir mapped to docker host directory (must have proper permissions)
|
||||
mv ../cassandra[-_]*${CASSANDRA_VERSION}* "${DIST_DIR}"
|
||||
# clean build deps
|
||||
rm -f cassandra-build-deps_*
|
||||
# restore debian/changelog
|
||||
git restore debian/changelog
|
||||
git restore debian/changelog || true
|
||||
|
||||
popd >/dev/null
|
||||
|
|
@ -104,7 +104,7 @@ else
|
|||
# from the branch name. In this case, fall back to version specified in build.xml.
|
||||
CASSANDRA_VERSION="${buildxml_version}"
|
||||
dt=`date +"%Y%m%d"`
|
||||
ref=`git rev-parse --short HEAD`
|
||||
ref=`git rev-parse --short HEAD || grep -q GitSHA src/resources/org/apache/cassandra/config/version.properties && grep GitSHA src/resources/org/apache/cassandra/config/version.properties | cut -d"=" -f2 || echo unknown`
|
||||
CASSANDRA_REVISION="${dt}git${ref}"
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -44,4 +44,5 @@ fi
|
|||
# these happen when the image hasn't pre-downloaded all the ccm versions used in tests
|
||||
rm -rf /tmp/ccm-*.tar.gz
|
||||
|
||||
set -x
|
||||
exit ${status}
|
||||
|
|
@ -95,15 +95,20 @@ image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}"
|
|||
|
||||
# Look for existing docker image, otherwise build
|
||||
if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
|
||||
# try docker login to increase dockerhub rate limits
|
||||
timeout -k 5 5 docker login >/dev/null
|
||||
echo "Build image not found locally, pulling image ${image_name}..."
|
||||
if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
|
||||
# Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
|
||||
echo "Building docker image..."
|
||||
until docker build -t ${image_name} -f docker/${dockerfile} . ; do
|
||||
echo "docker build failed… trying again in 10s… "
|
||||
sleep 10
|
||||
done
|
||||
echo "Docker image ${image_name} has been built"
|
||||
else
|
||||
echo "Successfully pulled build image."
|
||||
fi
|
||||
else
|
||||
echo "Found build image locally."
|
||||
fi
|
||||
|
||||
# Run build script through docker
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ ENV BUILD_HOME=/home/build
|
|||
ENV RPM_BUILD_DIR=$BUILD_HOME/rpmbuild
|
||||
ENV DIST_DIR=/dist
|
||||
ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
|
||||
ENV ANT_VERSION=1.10.12
|
||||
ENV ANT_VERSION=1.10.14
|
||||
ARG UID_ARG=1000
|
||||
ARG GID_ARG=1000
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,12 @@ if [ "$#" -lt 1 ] || [ "$#" -gt 3 ] || [ "$1" == "-h" ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
error() {
|
||||
echo >&2 $2;
|
||||
set -x
|
||||
exit $1
|
||||
}
|
||||
|
||||
# variables, with defaults
|
||||
[ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname "$0")/../..)"
|
||||
[ "x${cassandra_dtest_dir}" != "x" ] || cassandra_dtest_dir="${cassandra_dir}/../cassandra-dtest"
|
||||
|
|
@ -39,12 +45,12 @@ fi
|
|||
[ -d "${m2_dir}" ] || { mkdir -p "${m2_dir}" ; }
|
||||
|
||||
# pre-conditions
|
||||
command -v docker >/dev/null 2>&1 || { echo >&2 "docker needs to be installed"; exit 1; }
|
||||
command -v bc >/dev/null 2>&1 || { echo >&2 "bc needs to be installed"; exit 1; }
|
||||
command -v timeout >/dev/null 2>&1 || { echo >&2 "timeout needs to be installed"; exit 1; }
|
||||
(docker info >/dev/null 2>&1) || { echo >&2 "docker needs to running"; exit 1; }
|
||||
[ -f "${cassandra_dir}/build.xml" ] || { echo >&2 "${cassandra_dir}/build.xml must exist"; exit 1; }
|
||||
[ -f "${cassandra_dir}/.build/run-tests.sh" ] || { echo >&2 "${cassandra_dir}/.build/run-tests.sh must exist"; exit 1; }
|
||||
command -v docker >/dev/null 2>&1 || { error 1 "docker needs to be installed"; }
|
||||
command -v bc >/dev/null 2>&1 || { error 1 "bc needs to be installed"; }
|
||||
command -v timeout >/dev/null 2>&1 || { error 1 "timeout needs to be installed"; }
|
||||
(docker info >/dev/null 2>&1) || { error 1 "docker needs to running"; }
|
||||
[ -f "${cassandra_dir}/build.xml" ] || { error 1 "${cassandra_dir}/build.xml must exist"; }
|
||||
[ -f "${cassandra_dir}/.build/run-tests.sh" ] || { error 1 "${cassandra_dir}/.build/run-tests.sh must exist"; }
|
||||
|
||||
# arguments
|
||||
target=$1
|
||||
|
|
@ -63,8 +69,7 @@ fi
|
|||
|
||||
regx_java_version="(${java_version_supported//,/|})"
|
||||
if [[ ! "${java_version}" =~ $regx_java_version ]]; then
|
||||
echo "Error: Java version is not in ${java_version_supported}, it is set to ${java_version}"
|
||||
exit 1
|
||||
error 1 "Error: Java version is not in ${java_version_supported}, it is set to ${java_version}"
|
||||
fi
|
||||
|
||||
# allow python version override, otherwise default to current python version or 3.8
|
||||
|
|
@ -88,14 +93,10 @@ docker_mounts="${docker_mounts} -v "${build_dir}/tmp":/home/cassandra/cassandra/
|
|||
|
||||
# Look for existing docker image, otherwise build
|
||||
if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
|
||||
echo "Build image not found locally"
|
||||
# try docker login to increase dockerhub rate limits
|
||||
echo "Attempting 'docker login' to increase dockerhub rate limits"
|
||||
timeout -k 5 5 docker login >/dev/null 2>/dev/null
|
||||
echo "Pulling build image..."
|
||||
echo "Build image not found locally, pulling image ${image_name}..."
|
||||
if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
|
||||
# Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
|
||||
echo "Building docker image ${image_name}..."
|
||||
echo "Building docker image..."
|
||||
until docker build -t ${image_name} -f docker/${dockerfile} . ; do
|
||||
echo "docker build failed… trying again in 10s… "
|
||||
sleep 10
|
||||
|
|
@ -103,9 +104,9 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
|
|||
echo "Docker image ${image_name} has been built"
|
||||
else
|
||||
echo "Successfully pulled build image."
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "Found build image locally."
|
||||
echo "Found build image locally."
|
||||
fi
|
||||
|
||||
pushd ${cassandra_dir} >/dev/null
|
||||
|
|
@ -119,47 +120,47 @@ if [[ ! -z ${JENKINS_URL+x} ]] && [[ ! -z ${NODE_NAME+x} ]] ; then
|
|||
fi
|
||||
|
||||
# find host's available cores and mem
|
||||
cores=$(docker run --rm alpine nproc --all) || { echo >&2 "Unable to check available CPU cores"; exit 1; }
|
||||
cores=$(docker run --rm alpine:3.19.1 nproc --all) || { error 1 "Unable to check available CPU cores"; }
|
||||
|
||||
case $(uname) in
|
||||
"Linux")
|
||||
mem=$(docker run --rm alpine free -b | grep Mem: | awk '{print $2}') || { echo >&2 "Unable to check available memory"; exit 1; }
|
||||
mem=$(docker run --rm alpine:3.19.1 free -b | grep Mem: | awk '{print $2}') || { error 1 "Unable to check available memory"; }
|
||||
;;
|
||||
"Darwin")
|
||||
mem=$(sysctl -n hw.memsize) || { echo >&2 "Unable to check available memory"; exit 1; }
|
||||
mem=$(sysctl -n hw.memsize) || { error 1 "Unable to check available memory"; }
|
||||
;;
|
||||
*)
|
||||
echo >&2 "Unsupported operating system, expected Linux or Darwin"
|
||||
exit 1
|
||||
error 1 "Unsupported operating system, expected Linux or Darwin"
|
||||
esac
|
||||
|
||||
# figure out resource limits, scripts, and mounts for the test type
|
||||
docker_flags="-m 5g --memory-swap 5g"
|
||||
case ${target} in
|
||||
"build_dtest_jars")
|
||||
;;
|
||||
"stress-test" | "fqltool-test" )
|
||||
[[ ${mem} -gt $((1 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 1g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
|
||||
[[ ${mem} -gt $((1 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 1g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
|
||||
;;
|
||||
# test-burn doesn't have enough tests in it to split beyond 8, and burn and long we want a bit more resources anyway
|
||||
"microbench" | "test-burn" | "long-test" | "cqlsh-test" )
|
||||
[[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
|
||||
[[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
|
||||
;;
|
||||
"dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" )
|
||||
[ -f "${cassandra_dtest_dir}/dtest.py" ] || { echo >&2 "${cassandra_dtest_dir}/dtest.py must exist"; exit 1; }
|
||||
[[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
|
||||
"simulator-dtest")
|
||||
[[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
|
||||
docker_flags="-m 15g --memory-swap 15g"
|
||||
;;
|
||||
"dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large")
|
||||
[ -f "${cassandra_dtest_dir}/dtest.py" ] || { error 1 "${cassandra_dtest_dir}/dtest.py not found. please specify 'cassandra_dtest_dir' to point to the local cassandra-dtest source"; }
|
||||
test_script="run-python-dtests.sh"
|
||||
docker_mounts="${docker_mounts} -v ${cassandra_dtest_dir}:/home/cassandra/cassandra-dtest"
|
||||
# check that ${cassandra_dtest_dir} is valid
|
||||
[ -f "${cassandra_dtest_dir}/dtest.py" ] || { echo >&2 "${cassandra_dtest_dir}/dtest.py not found. please specify 'cassandra_dtest_dir' to point to the local cassandra-dtest source"; exit 1; }
|
||||
[[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
|
||||
docker_flags="-m 15g --memory-swap 15g"
|
||||
;;
|
||||
"test"| "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode" | "simulator-dtest")
|
||||
[[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { echo >&2 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; exit 1; }
|
||||
max_docker_runs_by_cores=$( echo "sqrt( ${cores} / ${jenkins_executors} )" | bc )
|
||||
max_docker_runs_by_mem=$(( ${mem} / ( 5 * 1024 * 1024 * 1024 * ${jenkins_executors} ) ))
|
||||
"test"| "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
|
||||
[[ ${mem} -gt $((5 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 6g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
|
||||
;;
|
||||
*)
|
||||
echo "unrecognized test type \"${target}\""
|
||||
exit 1
|
||||
error 1 "unrecognized test type \"${target}\""
|
||||
;;
|
||||
esac
|
||||
|
||||
|
|
@ -171,12 +172,8 @@ if (( $(echo "${docker_cpus} > ${docker_cpus_limit}" |bc -l) )) ; then
|
|||
fi
|
||||
|
||||
# hack: long-test does not handle limited CPUs
|
||||
if [ "${target}" == "long-test" ] ; then
|
||||
docker_flags="-m 5g --memory-swap 5g"
|
||||
elif [[ "${target}" =~ dtest* ]] ; then
|
||||
docker_flags="--cpus=${docker_cpus} -m 15g --memory-swap 15g"
|
||||
else
|
||||
docker_flags="--cpus=${docker_cpus} -m 5g --memory-swap 5g"
|
||||
if [ "${target}" != "long-test" ] ; then
|
||||
docker_flags="--cpus=${docker_cpus} ${docker_flags}"
|
||||
fi
|
||||
|
||||
docker_flags="${docker_flags} -d --rm"
|
||||
|
|
@ -203,7 +200,7 @@ esac
|
|||
|
||||
# cython can be used for cqlsh-test
|
||||
if [ "$cython" == "yes" ]; then
|
||||
[ "${target}" == "cqlsh-test" ] || { echo "cython is only supported for cqlsh-test"; exit 1; }
|
||||
[ "${target}" == "cqlsh-test" ] || { error 1 "cython is only supported for cqlsh-test"; }
|
||||
ANT_OPTS="${ANT_OPTS}_cython"
|
||||
else
|
||||
cython="no"
|
||||
|
|
@ -268,4 +265,5 @@ xz -f ${logfile} 2>/dev/null
|
|||
|
||||
popd >/dev/null
|
||||
popd >/dev/null
|
||||
set -x
|
||||
exit ${status}
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ RUN echo "cassandra-tmp ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/build
|
|||
RUN chmod 0440 /etc/sudoers.d/build
|
||||
|
||||
# switch to the cassandra user
|
||||
RUN mkdir -p ${BUILD_HOME} && chmod a+rwx ${BUILD_HOME}
|
||||
USER cassandra-tmp
|
||||
ENV HOME ${BUILD_HOME}
|
||||
WORKDIR ${BUILD_HOME}
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ _main() {
|
|||
fi
|
||||
|
||||
# check project is already built. no cleaning is done, so jenkins unstash works, beware.
|
||||
[[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR})"; exit 1; }
|
||||
[[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f "${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls ${DIST_DIR} | xargs)"; exit 1; }
|
||||
|
||||
# check if dist artifacts exist, this breaks the dtests
|
||||
[[ -d "${DIST_DIR}/dist" ]] && { echo "tests don't work when build/dist ("${DIST_DIR}/dist") exists (from \`ant artifacts\`)"; exit 1; }
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@
|
|||
// will be ignored when run on other environments.
|
||||
// Note there are also differences when CI is being run pre- or post-commit.
|
||||
//
|
||||
// CAUTION! When running CI with changes in this file, ensure the "Pipeline script from SCM" scm details match
|
||||
// the brances being tested. These details don't honour the per-build repository and branch parameterisation.
|
||||
//
|
||||
// Validate/lint this file using the following command
|
||||
// `curl -X POST -F "jenkinsfile=<.jenkins/Jenkinsfile" https://ci-cassandra.apache.org/pipeline-model-converter/validate`
|
||||
|
|
@ -56,18 +58,22 @@
|
|||
|
||||
pipeline {
|
||||
agent { label 'cassandra-small' }
|
||||
options {
|
||||
// must have: avoids agents waste in idle time on controller bottleneck
|
||||
durabilityHint('PERFORMANCE_OPTIMIZED')
|
||||
}
|
||||
parameters {
|
||||
string(name: 'repository', defaultValue: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository')
|
||||
string(name: 'branch', defaultValue: env.BRANCH_NAME, description: 'Branch')
|
||||
string(name: 'repository', defaultValue: params.repository ?: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository')
|
||||
string(name: 'branch', defaultValue: params.branch ?: scm.userRemoteConfigs[0].refspec, description: 'Branch')
|
||||
|
||||
choice(name: 'profile', choices: pipelineProfiles().keySet() as List, description: 'Pick a pipeline profile.')
|
||||
string(name: 'profile_custom_regexp', defaultValue: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*')
|
||||
choice(name: 'profile', choices: pipelineProfileNames(params.profile ?: ''), description: 'Pick a pipeline profile.')
|
||||
string(name: 'profile_custom_regexp', defaultValue: params.profile_custom_regexp ?: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*')
|
||||
|
||||
choice(name: 'architecture', choices: archsSupported() + "all", description: 'Pick architecture. The ARM64 is disabled by default at the moment.')
|
||||
string(name: 'jdk', defaultValue: "", description: 'Restrict JDK versions. (e.g. "11", "17", etc)')
|
||||
string(name: 'jdk', defaultValue: params.jdk ?: '', description: 'Restrict JDK versions. (e.g. "11", "17", etc)')
|
||||
|
||||
string(name: 'dtest_repository', defaultValue: 'https://github.com/apache/cassandra-dtest' ,description: 'Cassandra DTest Repository')
|
||||
string(name: 'dtest_branch', defaultValue: 'trunk', description: 'DTest Branch')
|
||||
string(name: 'dtest_repository', defaultValue: params.dtest_repository ?: 'https://github.com/apache/cassandra-dtest', description: 'Cassandra DTest Repository')
|
||||
string(name: 'dtest_branch', defaultValue: params.dtest_branch ?: 'trunk', description: 'DTest Branch')
|
||||
}
|
||||
stages {
|
||||
stage('jar') {
|
||||
|
|
@ -125,6 +131,13 @@ def pipelineProfiles() {
|
|||
]
|
||||
}
|
||||
|
||||
def pipelineProfileNames(putFirst) {
|
||||
set = pipelineProfiles().keySet() as List
|
||||
set = set - putFirst
|
||||
set.add(0, putFirst)
|
||||
return set
|
||||
}
|
||||
|
||||
def tasks() {
|
||||
// Steps config
|
||||
def buildSteps = [
|
||||
|
|
@ -152,9 +165,9 @@ def tasks() {
|
|||
'long-test': [splits: 8],
|
||||
'test-oa': [splits: 8],
|
||||
'test-system-keyspace-directory': [splits: 8],
|
||||
'jvm-dtest': [splits: 8],
|
||||
'jvm-dtest': [splits: 12],
|
||||
'jvm-dtest-upgrade': [splits: 8],
|
||||
'simulator-dtest': [splits: 1],
|
||||
'simulator-dtest': [splits: 1, size: 'large'],
|
||||
'dtest': [splits: 64, size: 'large'],
|
||||
'dtest-novnode': [splits: 64, size: 'large'],
|
||||
'dtest-latest': [splits: 64, size: 'large'],
|
||||
|
|
@ -221,7 +234,7 @@ def tasks() {
|
|||
|
||||
@NonCPS
|
||||
def List getMatrixAxes(Map matrix_axes) {
|
||||
List axes = []
|
||||
def List axes = []
|
||||
matrix_axes.each { axis, values ->
|
||||
List axisList = []
|
||||
values.each { value ->
|
||||
|
|
@ -233,10 +246,10 @@ def List getMatrixAxes(Map matrix_axes) {
|
|||
}
|
||||
|
||||
def getStepName(cell, command) {
|
||||
arch = "amd64" == cell.arch ? "" : " ${cell.arch}"
|
||||
python = "cqlsh-test" != cell.step ? "" : " python${cell.python}"
|
||||
cython = "no" == cell.cython ? "" : " cython"
|
||||
split = command.splits > 1 ? " ${cell.split}/${command.splits}" : ""
|
||||
def arch = "amd64" == cell.arch ? "" : " ${cell.arch}"
|
||||
def python = "cqlsh-test" != cell.step ? "" : " python${cell.python}"
|
||||
def cython = "no" == cell.cython ? "" : " cython"
|
||||
def split = command.splits > 1 ? " ${cell.split}/${command.splits}" : ""
|
||||
return "${cell.step}${arch} jdk${cell.jdk}${python}${cython}${split}"
|
||||
}
|
||||
|
||||
|
|
@ -286,7 +299,7 @@ def isCanonical() {
|
|||
}
|
||||
|
||||
def isStageEnabled(stage) {
|
||||
return "jar" == stage || pipelineProfiles()[params.profile].contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp)
|
||||
return "jar" == stage || pipelineProfiles()[params.profile]?.contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp)
|
||||
}
|
||||
|
||||
def isArchEnabled(arch) {
|
||||
|
|
@ -311,26 +324,24 @@ def build(command, cell) {
|
|||
nodeExclusion = "&&!${NODE_NAME}"
|
||||
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
|
||||
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
|
||||
cleanAgent(cell.step)
|
||||
cleanWs()
|
||||
fetchSource(cell.step, cell.arch, cell.jdk)
|
||||
sh """
|
||||
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
|
||||
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
|
||||
"""
|
||||
"""
|
||||
fetchDockerImages(['almalinux-build', 'bullseye-build', 'centos7-build'])
|
||||
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
|
||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log"
|
||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
||||
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
|
||||
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
||||
status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee build/${logfile}", returnStatus: true
|
||||
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
||||
dir("build") {
|
||||
sh "xz -f *${logfile}"
|
||||
archiveArtifacts artifacts: "${logfile}.xz", fingerprint: true
|
||||
copyToNightlies("${logfile}.xz", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
||||
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
||||
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
||||
}
|
||||
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
||||
if ("jar" == cell.step) { // TODO only stash the project built files. all dependency libraries are restored from the local maven repo using `ant resolver-dist-lib`
|
||||
stash name: "${cell.arch}_${cell.jdk}", useDefaultExcludes: false //, includes: '**/*.jar' //, includes: "*.jar,classes/**,test/classes/**,tools/**"
|
||||
stash name: "${cell.arch}_${cell.jdk}"
|
||||
}
|
||||
dir("build") {
|
||||
copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
||||
|
|
@ -353,11 +364,10 @@ def test(command, cell) {
|
|||
nodeExclusion = "&&!${NODE_NAME}"
|
||||
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
|
||||
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
|
||||
cleanAgent(cell.step)
|
||||
cleanWs()
|
||||
fetchSource(cell.step, cell.arch, cell.jdk)
|
||||
fetchDockerImages(['ubuntu2004_test'])
|
||||
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
|
||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log"
|
||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
||||
// pipe to tee needs pipefail
|
||||
def script_vars = "#!/bin/bash \n set -o pipefail ; "
|
||||
script_vars = "${script_vars} python_version=\'${cell.python}\'"
|
||||
|
|
@ -367,11 +377,10 @@ def test(command, cell) {
|
|||
}
|
||||
script_vars = fetchDTestsSource(command, script_vars)
|
||||
buildJVMDTestJars(cell, script_vars, logfile)
|
||||
status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee -a build/${logfile}", returnStatus: true
|
||||
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
||||
dir("build") {
|
||||
sh "xz -f ${logfile}"
|
||||
archiveArtifacts artifacts: "${logfile}.xz", fingerprint: true
|
||||
copyToNightlies("${logfile}.xz", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
|
||||
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
||||
copyToNightlies("${logfile}", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
|
||||
}
|
||||
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
||||
dir("build") {
|
||||
|
|
@ -394,17 +403,18 @@ def test(command, cell) {
|
|||
}
|
||||
|
||||
def fetchSource(stage, arch, jdk) {
|
||||
if ("jar" == stage) {
|
||||
checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
|
||||
sh "mkdir -p build/stage-logs"
|
||||
} else {
|
||||
unstash name: "${arch}_${jdk}"
|
||||
}
|
||||
cleanAgent(stage)
|
||||
if ("jar" == stage) {
|
||||
checkout changelog: false, scm: scmGit(branches: [[name: params.branch ?: 'trunk']], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
|
||||
sh "mkdir -p build/stage-logs"
|
||||
} else {
|
||||
unstash name: "${arch}_${jdk}"
|
||||
}
|
||||
}
|
||||
|
||||
def fetchDTestsSource(command, script_vars) {
|
||||
if (command.containsKey('python-dtest')) {
|
||||
checkout changelog: false, poll: false, scm: scmGit(branches: [[name: params.dtest_branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true), [$class: 'RelativeTargetDirectory', relativeTargetDir: "${WORKSPACE}/build/cassandra-dtest"]], userRemoteConfigs: [[url: params.dtest_repository]])
|
||||
checkout changelog: false, poll: false, scm: scmGit(branches: [[name: params.dtest_branch ?: 'trunk']], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true), [$class: 'RelativeTargetDirectory', relativeTargetDir: "${WORKSPACE}/build/cassandra-dtest"]], userRemoteConfigs: [[url: params.dtest_repository]])
|
||||
sh "test -f build/cassandra-dtest/requirements.txt || { echo 'Invalid cassandra-dtest fork/branch'; exit 1; }"
|
||||
return "${script_vars} cassandra_dtest_dir='${WORKSPACE}/build/cassandra-dtest'"
|
||||
}
|
||||
|
|
@ -416,12 +426,29 @@ def buildJVMDTestJars(cell, script_vars, logfile) {
|
|||
try {
|
||||
unstash name: "jvm_dtests_${cell.arch}_${cell.jdk}"
|
||||
} catch (error) {
|
||||
sh label: "RUNNING build_dtest_jars...", script: "${script_vars} .build/docker/run-tests.sh build_dtest_jars ${cell.jdk} 2>&1 | tee build/${logfile}"
|
||||
sh label: "RUNNING build_dtest_jars...", script: "${script_vars} .build/docker/run-tests.sh build_dtest_jars ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )"
|
||||
stash name: "jvm_dtests_${cell.arch}_${cell.jdk}", includes: '**/dtest*.jar'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def fetchDockerImages(dockerfiles) {
|
||||
// prefetch, from apache jfrog, reduces risking dockerhub pull rate limits
|
||||
// also prefetch alpine:latest as its used as a utility in the scripts
|
||||
def dockerfilesVar = dockerfiles.join(' ')
|
||||
sh """#!/bin/bash
|
||||
for dockerfile in ${dockerfilesVar} ; do
|
||||
image_tag="\$(md5sum .build/docker/\${dockerfile}.docker | cut -d' ' -f1)"
|
||||
image_name="apache/cassandra-\${dockerfile}:\${image_tag}"
|
||||
if ! ( [[ "" != "\$(docker images -q \${image_name} 2>/dev/null)" ]] ) ; then
|
||||
docker pull -q apache.jfrog.io/cassan-docker/\${image_name} &
|
||||
fi
|
||||
done
|
||||
docker pull -q apache.jfrog.io/cassan-docker/alpine:3.19.1 &
|
||||
wait
|
||||
"""
|
||||
}
|
||||
|
||||
def getNodeLabel(command, cell) {
|
||||
echo "using node label: cassandra-${cell.arch}-${command.size}"
|
||||
return "cassandra-${cell.arch}-${command.size}"
|
||||
|
|
@ -434,13 +461,13 @@ def copyToNightlies(sourceFiles, remoteDirectory='') {
|
|||
retry(9) {
|
||||
if (attempt > 1) { sleep(60 * attempt) }
|
||||
sshPublisher(
|
||||
continueOnError: true, failOnError: false,
|
||||
publishers: [
|
||||
sshPublisherDesc(
|
||||
configName: "Nightlies",
|
||||
transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ]
|
||||
)
|
||||
])
|
||||
continueOnError: true, failOnError: false,
|
||||
publishers: [
|
||||
sshPublisherDesc(
|
||||
configName: "Nightlies",
|
||||
transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ]
|
||||
)
|
||||
])
|
||||
}
|
||||
echo "archived to https://nightlies.apache.org/${remotePath}"
|
||||
}
|
||||
|
|
@ -449,13 +476,32 @@ def copyToNightlies(sourceFiles, remoteDirectory='') {
|
|||
def cleanAgent(job_name) {
|
||||
sh "hostname"
|
||||
if (isCanonical()) {
|
||||
def maxJobHours = 12
|
||||
echo "Cleaning project, and pruning docker for '${job_name}' on ${NODE_NAME}…" ;
|
||||
sh """
|
||||
git clean -qxdff -e build/test/jmh-result.json || true;
|
||||
if pgrep -xa docker || pgrep -af "build/docker" || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --all --force --filter "until=${maxJobHours}h" || true ; else docker system prune --force --volumes || true ; fi;
|
||||
"""
|
||||
def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/"
|
||||
cleanAgentDocker(job_name, agentScriptsUrl)
|
||||
logAgentInfo(job_name, agentScriptsUrl)
|
||||
}
|
||||
cleanWs()
|
||||
}
|
||||
|
||||
def cleanAgentDocker(job_name, agentScriptsUrl) {
|
||||
// we don't expect any build to have been running for longer than maxBuildHours
|
||||
def maxBuildHours = 12
|
||||
echo "Pruning docker for '${job_name}' on ${NODE_NAME}…" ;
|
||||
sh """#!/bin/bash
|
||||
set +e
|
||||
wget -q ${agentScriptsUrl}/docker_image_pruner.py
|
||||
wget -q ${agentScriptsUrl}/docker_agent_cleaner.sh
|
||||
bash docker_agent_cleaner.sh ${maxBuildHours}
|
||||
"""
|
||||
}
|
||||
|
||||
def logAgentInfo(job_name, agentScriptsUrl) {
|
||||
sh """#!/bin/bash
|
||||
set +e -o pipefail
|
||||
wget -q ${agentScriptsUrl}/agent_report.sh
|
||||
bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt
|
||||
"""
|
||||
copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/")
|
||||
}
|
||||
|
||||
/////////////////////////////////////////
|
||||
|
|
@ -463,18 +509,29 @@ def cleanAgent(job_name) {
|
|||
/////////////////////////////////////////
|
||||
|
||||
def generateTestReports() {
|
||||
// built-in on ci-cassandra will be much faster (local transfer for copyArtifacts and archiveArtifacts)
|
||||
def nodeName = isCanonical() ? "built-in" : "cassandra-medium"
|
||||
node(nodeName) {
|
||||
cleanWs()
|
||||
node("cassandra-medium") {
|
||||
cleanAgent("generateTestReports")
|
||||
checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
|
||||
copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
|
||||
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_generateTestReports.log.xz"
|
||||
sh "mkdir -p build/stage-logs"
|
||||
def teeSuffix = "2>&1 | tee >( xz -c > build/${logfile} )"
|
||||
def script_vars = "#!/bin/bash -x \n "
|
||||
if (isCanonical()) {
|
||||
// copyArtifacts takes >4hrs, hack with manual download
|
||||
sh """${script_vars}
|
||||
( mkdir -p build/test
|
||||
wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip
|
||||
unzip -x -d build/test -q output.zip ) ${teeSuffix}
|
||||
"""
|
||||
} else {
|
||||
copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
|
||||
}
|
||||
if (fileExists('build/test/output')) {
|
||||
// merge splits for each target's test report, other axes are kept separate
|
||||
// TODO parallelised for loop
|
||||
// TODO results_details.tar.xz needs to include all logs for failed tests
|
||||
sh """
|
||||
find build/test/output -name *.xml.xz -exec sh -c 'xz -f --decompress {} &' ';' ; wait
|
||||
sh """${script_vars} (
|
||||
find build/test/output -name *.xml.xz -exec sh -c 'xz -f --decompress {} &' ';' ; wait
|
||||
|
||||
for target in \$(ls build/test/output/) ; do
|
||||
if test -d build/test/output/\${target} ; then
|
||||
|
|
@ -488,12 +545,13 @@ def generateTestReports() {
|
|||
|
||||
.build/docker/_docker_run.sh bullseye-build.docker ci/generate-ci-summary.sh || echo "failed generate-ci-summary.sh"
|
||||
|
||||
tar -cf build/results_details.tar -C build/test/ reports && xz -9f build/results_details.tar
|
||||
tar -cf build/results_details.tar -C build/test/ reports
|
||||
xz -8f build/results_details.tar ) ${teeSuffix}
|
||||
"""
|
||||
|
||||
dir('build/') {
|
||||
archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz", fingerprint: true
|
||||
copyToNightlies('results_details.tar.xz')
|
||||
archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true
|
||||
copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile}')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -502,6 +560,7 @@ def generateTestReports() {
|
|||
def sendNotifications() {
|
||||
if (isPostCommit() && isCanonical()) {
|
||||
// the following is expected only to work on ci-cassandra.apache.org
|
||||
def changes = '?'
|
||||
try {
|
||||
script {
|
||||
changes = formatChangeLogChanges(currentBuild.changeSets)
|
||||
|
|
|
|||
|
|
@ -1851,7 +1851,7 @@
|
|||
</fileset>
|
||||
<report todir="${build.test.report.dir}/unitTestReport/html" unless:true="${print-summary.skip}" />
|
||||
</junitreport>
|
||||
<!-- concat the report through a filter chain to extract what you want -->
|
||||
<!-- concat the report through a filter chain to extract a single line summary -->
|
||||
<concat unless:true="${print-summary.skip}">
|
||||
<fileset file="${build.test.report.dir}/unitTestReport/html/overview-summary.html" />
|
||||
<filterchain>
|
||||
|
|
|
|||
Loading…
Reference in New Issue