mirror of https://github.com/apache/cassandra
604 lines
26 KiB
Groovy
604 lines
26 KiB
Groovy
#!/usr/bin/env groovy
|
|
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
//
|
|
//
|
|
// Jenkins CI declaration.
|
|
//
|
|
// The declarative pipeline is presented first as a high level view.
|
|
//
|
|
// Build and Test Stages are dynamic, the full possible list defined by the `tasks()` function.
|
|
// There is a choice of pipeline profles with sets of tasks that are run, see `pipelineProfiles()`.
|
|
//
|
|
// All tasks use the dockerised CI-agnostic scripts found under `.build/docker/`
|
|
// The `type: test` always `.build/docker/run-tests.sh`
|
|
//
|
|
//
|
|
// This Jenkinsfile is expected to work on any Jenkins infrastructure.
|
|
// The controller should have 4 cpu, 12GB ram (and be configured to use `-XX:+UseG1GC -Xmx8G`)
|
|
// It is required to have agents providing five labels, each that can provide docker and the following capabilities:
|
|
// - cassandra-amd64-small : 1 cpu, 1GB ram
|
|
// - cassandra-small : 1 cpu, 1GB ram (alias for above but for any arch)
|
|
// - cassandra-amd64-medium : 3 cpu, 5GB ram
|
|
// - cassandra-medium : 3 cpu, 5GB ram (alias for above but for any arch)
|
|
// - cassandra-amd64-large : 7 cpu, 14GB ram
|
|
//
|
|
// When running builds parameterised to other architectures the corresponding labels are expected.
|
|
// For example 'arm64' requires the labels: cassandra-arm64-small, cassandra-arm64-medium, cassandra-arm64-large.
|
|
//
|
|
// The built-in node must has the "controller" label. There must be more than two agents for each label.
|
|
//
|
|
// Plugins required are:
|
|
// git, workflow-job, workflow-cps, junit, workflow-aggregator, ws-cleanup, pipeline-build-step, test-stability, copyartifact.
|
|
//
|
|
// Any functionality that depends upon ASF Infra ( i.e. the canonical ci-cassandra.a.o )
|
|
// will be ignored when run on other environments.
|
|
// Note there are also differences when CI is being run pre- or post-commit.
|
|
//
|
|
// CAUTION! When running CI with changes in this file, ensure the "Pipeline script from SCM" scm details match
|
|
// the brances being tested. These details don't honour the per-build repository and branch parameterisation.
|
|
//
|
|
// Validate/lint this file using the following command
|
|
// `curl -X POST -F "jenkinsfile=<.jenkins/Jenkinsfile" https://ci-cassandra.apache.org/pipeline-model-converter/validate`
|
|
//
|
|
|
|
|
|
pipeline {
|
|
agent { label 'cassandra-small' }
|
|
options {
|
|
// must have: avoids agents waste in idle time on controller bottleneck
|
|
durabilityHint('PERFORMANCE_OPTIMIZED')
|
|
}
|
|
parameters {
|
|
string(name: 'repository', defaultValue: params.repository ?: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository')
|
|
string(name: 'branch', defaultValue: params.branch ?: scm.userRemoteConfigs[0].refspec, description: 'Branch')
|
|
|
|
choice(name: 'profile', choices: pipelineProfileNames(params.profile ?: ''), description: 'Pick a pipeline profile.')
|
|
string(name: 'profile_custom_regexp', defaultValue: params.profile_custom_regexp ?: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*')
|
|
|
|
choice(name: 'architecture', choices: archsSupported() + "all", description: 'Pick architecture. The ARM64 is disabled by default at the moment.')
|
|
string(name: 'jdk', defaultValue: params.jdk ?: '', description: 'Restrict JDK versions. (e.g. "11", "17", etc)')
|
|
|
|
string(name: 'dtest_repository', defaultValue: params.dtest_repository ?: 'https://github.com/apache/cassandra-dtest', description: 'Cassandra DTest Repository')
|
|
string(name: 'dtest_branch', defaultValue: params.dtest_branch ?: 'trunk', description: 'DTest Branch')
|
|
}
|
|
stages {
|
|
stage('jar') {
|
|
// the jar stage executes only the 'jar' build step, via the build(…) function
|
|
// the results of these (per jdk, per arch) are then stashed and used for every other build and test step
|
|
steps {
|
|
script {
|
|
parallel(getJarTasks())
|
|
}
|
|
}
|
|
}
|
|
stage('Tests') {
|
|
// the Tests stage executes all other build and task steps.
|
|
// build steps are sent to the build(…) function, test steps sent to the test(…) function
|
|
// these steps are parameterised and split by the tasks() function
|
|
when {
|
|
expression { hasNonJarTasks() }
|
|
}
|
|
steps {
|
|
script {
|
|
parallel(tasks()['tests'])
|
|
}
|
|
}
|
|
}
|
|
stage('Summary') {
|
|
// generate the ci_summary.html and results_details.tar.xz artefacts
|
|
steps {
|
|
generateTestReports()
|
|
}
|
|
}
|
|
}
|
|
post {
|
|
always {
|
|
sendNotifications()
|
|
}
|
|
}
|
|
}
|
|
|
|
///////////////////////////
|
|
//// scripting support ////
|
|
///////////////////////////
|
|
|
|
def archsSupported() { return ["amd64", "arm64"] }
|
|
def pythonsSupported() { return ["3.8", "3.11"] }
|
|
def pythonDefault() { return "3.8" }
|
|
|
|
def pipelineProfiles() {
|
|
return [
|
|
'packaging': ['artifacts', 'lint', 'debian', 'redhat'],
|
|
'skinny': ['lint', 'cqlsh-test', 'test', 'jvm-dtest', 'simulator-dtest', 'dtest'],
|
|
'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest'],
|
|
'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade'],
|
|
'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-novnode-large'],
|
|
'custom': []
|
|
]
|
|
}
|
|
|
|
def pipelineProfileNames(putFirst) {
|
|
set = pipelineProfiles().keySet() as List
|
|
set = set - putFirst
|
|
set.add(0, putFirst)
|
|
return set
|
|
}
|
|
|
|
def tasks() {
|
|
// Steps config
|
|
def buildSteps = [
|
|
'jar': [script: 'build-jars.sh', toCopy: null],
|
|
'artifacts': [script: 'build-artifacts.sh', toCopy: 'apache-cassandra-*.tar.gz,apache-cassandra-*.jar,apache-cassandra-*.pom'],
|
|
'lint': [script: 'check-code.sh', toCopy: null],
|
|
'debian': [script: 'build-debian.sh', toCopy: 'cassandra_*,cassandra-tools_*'],
|
|
'redhat': [script: 'build-redhat.sh rpm', toCopy: '*.rpm'],
|
|
]
|
|
buildSteps.each() {
|
|
it.value.put('type', 'build')
|
|
it.value.put('size', 'small')
|
|
it.value.put('splits', 1)
|
|
}
|
|
|
|
def testSteps = [
|
|
'cqlsh-test': [splits: 1],
|
|
'fqltool-test': [splits: 1, size: 'small'],
|
|
'test-cdc': [splits: 8],
|
|
'test': [splits: 8],
|
|
'test-latest': [splits: 8],
|
|
'test-compression': [splits: 8],
|
|
'stress-test': [splits: 1, size: 'small'],
|
|
'test-burn': [splits: 2],
|
|
'long-test': [splits: 8],
|
|
'test-oa': [splits: 8],
|
|
'test-system-keyspace-directory': [splits: 8],
|
|
'jvm-dtest': [splits: 12],
|
|
'jvm-dtest-upgrade': [splits: 8],
|
|
'simulator-dtest': [splits: 1, size: 'large'],
|
|
'dtest': [splits: 64, size: 'large'],
|
|
'dtest-novnode': [splits: 64, size: 'large'],
|
|
'dtest-latest': [splits: 64, size: 'large'],
|
|
'dtest-large': [splits: 8, size: 'large'],
|
|
'dtest-large-novnode': [splits: 8, size: 'large'],
|
|
'dtest-upgrade': [splits: 64, size: 'large'],
|
|
'dtest-upgrade-novnode': [splits: 64, size: 'large'],
|
|
'dtest-upgrade-large': [splits: 64, size: 'large'],
|
|
'dtest-upgrade-novnode-large': [splits: 64, size: 'large'],
|
|
]
|
|
testSteps.each() {
|
|
it.value.put('type', 'test')
|
|
it.value.put('script', '.build/docker/run-tests.sh')
|
|
if (!it.value['size']) {
|
|
it.value.put('size', 'medium')
|
|
}
|
|
if (it.key.startsWith('dtest')) {
|
|
it.value.put('python-dtest', true)
|
|
}
|
|
}
|
|
|
|
def stepsMap = buildSteps + testSteps
|
|
|
|
// define matrix axes
|
|
def Map matrix_axes = [
|
|
arch: archsSupported(),
|
|
jdk: javaVersionsSupported(),
|
|
python: pythonsSupported(),
|
|
cython: ['yes', 'no'],
|
|
step: stepsMap.keySet(),
|
|
split: (1..testSteps.values().splits.max()).toList()
|
|
]
|
|
|
|
def javaVersionDefault = javaVersionDefault()
|
|
|
|
def List _axes = getMatrixAxes(matrix_axes).findAll { axis ->
|
|
(isArchEnabled(axis['arch'])) && // skip disabled archs
|
|
(isJdkEnabled(axis['jdk'])) && // skip disabled jdks
|
|
(isStageEnabled(axis['step'])) && // skip disabled steps
|
|
!(axis['python'] != pythonDefault() && 'cqlsh-test' != axis['step']) && // Use only python 3.8 for all tests but cqlsh-test
|
|
!(axis['cython'] != 'no' && 'cqlsh-test' != axis['step']) && // cython only for cqlsh-test, disable for others
|
|
!(axis['jdk'] != javaVersionDefault && ('cqlsh-test' == axis['step'] || 'simulator-dtest' == axis['step'] || axis['step'].contains('dtest-upgrade'))) && // run cqlsh-test, simulator-dtest, *dtest-upgrade only with jdk11
|
|
// Disable splits for all but proper stages
|
|
!(axis['split'] > 1 && !stepsMap.findAll { entry -> entry.value.splits >= axis['split'] }.keySet().contains(axis['step'])) &&
|
|
// run only the build types on non-amd64
|
|
!(axis['arch'] != 'amd64' && stepsMap.findAll { entry -> 'build' == entry.value.type }.keySet().contains(axis['step']))
|
|
}
|
|
|
|
def Map tasks = [
|
|
jars: [failFast: true],
|
|
tests: [failFast: true]
|
|
]
|
|
|
|
for (def axis in _axes) {
|
|
def cell = axis
|
|
def name = getStepName(cell, stepsMap[cell.step])
|
|
tasks[cell.step == "jar" ? "jars" : "tests"][name] = { ->
|
|
"${stepsMap[cell.step].type}"(stepsMap[cell.step], cell)
|
|
}
|
|
}
|
|
|
|
return tasks
|
|
}
|
|
|
|
@NonCPS
|
|
def List getMatrixAxes(Map matrix_axes) {
|
|
def List axes = []
|
|
matrix_axes.each { axis, values ->
|
|
List axisList = []
|
|
values.each { value ->
|
|
axisList << [(axis): value]
|
|
}
|
|
axes << axisList
|
|
}
|
|
axes.combinations()*.sum()
|
|
}
|
|
|
|
def getStepName(cell, command) {
|
|
def arch = "amd64" == cell.arch ? "" : " ${cell.arch}"
|
|
def python = "cqlsh-test" != cell.step ? "" : " python${cell.python}"
|
|
def cython = "no" == cell.cython ? "" : " cython"
|
|
def split = command.splits > 1 ? " ${cell.split}/${command.splits}" : ""
|
|
return "${cell.step}${arch} jdk${cell.jdk}${python}${cython}${split}"
|
|
}
|
|
|
|
def getJarTasks() {
|
|
Map jars = tasks()['jars']
|
|
assertJarTasks(jars)
|
|
return jars
|
|
}
|
|
|
|
def assertJarTasks(jars) {
|
|
if (jars.size() < 2) {
|
|
error("Nothing to build. Check parameters: jdk ${params.jdk} (${javaVersionsSupported()}), arch ${params.architecture} (${archsSupported()})")
|
|
}
|
|
}
|
|
|
|
def hasNonJarTasks() {
|
|
return tasks()['tests'].size() > 1
|
|
}
|
|
|
|
/**
|
|
* Return the default JDK defined by build.xml
|
|
**/
|
|
def javaVersionDefault() {
|
|
sh (returnStdout: true, script: 'grep \'property\\s*name=\"java.default\"\' build.xml | sed -ne \'s/.*value=\"\\([^\"]*\\)\".*/\\1/p\'').trim()
|
|
}
|
|
|
|
/**
|
|
* Return the supported JDKs defined by build.xml
|
|
**/
|
|
def javaVersionsSupported() {
|
|
sh (returnStdout: true, script: 'grep \'property\\s*name=\"java.supported\"\' build.xml | sed -ne \'s/.*value=\"\\([^\"]*\\)\".*/\\1/p\'').trim().split(',')
|
|
}
|
|
|
|
/**
|
|
* Is this a post-commit build (or a pre-commit build)
|
|
**/
|
|
def isPostCommit() {
|
|
// any build of a branch found on github.com/apache/cassandra is considered a post-commit (post-merge) CI run
|
|
return params.repository && params.repository.contains("apache/cassandra") // no params exist first build
|
|
}
|
|
|
|
/**
|
|
* Are we running on ci-cassandra.apache.org ?
|
|
**/
|
|
def isCanonical() {
|
|
return "${JENKINS_URL}".contains("ci-cassandra.apache.org")
|
|
}
|
|
|
|
def isStageEnabled(stage) {
|
|
return "jar" == stage || pipelineProfiles()[params.profile]?.contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp)
|
|
}
|
|
|
|
def isArchEnabled(arch) {
|
|
return params.architecture == arch || "all" == params.architecture
|
|
}
|
|
|
|
def isJdkEnabled(jdk) {
|
|
return !params.jdk?.trim() || params.jdk.trim() == jdk
|
|
}
|
|
|
|
/**
|
|
* Renders build script into pipeline steps
|
|
**/
|
|
def build(command, cell) {
|
|
def build_script = ".build/docker/${command.script}"
|
|
def maxAttempts = 2
|
|
def attempt = 0
|
|
def nodeExclusion = ""
|
|
retry(maxAttempts) {
|
|
attempt++
|
|
node(getNodeLabel(command, cell) + nodeExclusion) {
|
|
nodeExclusion = "&&!${NODE_NAME}"
|
|
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
|
|
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
|
|
fetchSource(cell.step, cell.arch, cell.jdk)
|
|
sh """
|
|
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
|
|
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
|
|
"""
|
|
fetchDockerImages(['almalinux-build', 'bullseye-build', 'centos7-build'])
|
|
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
|
|
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
|
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
|
|
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
|
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
|
dir("build") {
|
|
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
|
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
|
}
|
|
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
|
if ("jar" == cell.step) { // TODO only stash the project built files. all dependency libraries are restored from the local maven repo using `ant resolver-dist-lib`
|
|
stash name: "${cell.arch}_${cell.jdk}"
|
|
}
|
|
dir("build") {
|
|
copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
|
|
}
|
|
cleanAgent(cell.step)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
def test(command, cell) {
|
|
def splits = command.splits ? command.splits : 1
|
|
def maxAttempts = 2
|
|
def attempt = 0
|
|
def nodeExclusion = ""
|
|
retry(maxAttempts) {
|
|
attempt++
|
|
node(getNodeLabel(command, cell) + nodeExclusion) {
|
|
nodeExclusion = "&&!${NODE_NAME}"
|
|
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
|
|
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
|
|
fetchSource(cell.step, cell.arch, cell.jdk)
|
|
fetchDockerImages(['ubuntu2004_test'])
|
|
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
|
|
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
|
|
// pipe to tee needs pipefail
|
|
def script_vars = "#!/bin/bash \n set -o pipefail ; "
|
|
script_vars = "${script_vars} python_version=\'${cell.python}\'"
|
|
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
|
|
if ("cqlsh-test" == cell.step) {
|
|
script_vars = "${script_vars} cython=\'${cell.cython}\'"
|
|
}
|
|
script_vars = fetchDTestsSource(command, script_vars)
|
|
buildJVMDTestJars(cell, script_vars, logfile)
|
|
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
|
|
dir("build") {
|
|
archiveArtifacts artifacts: "${logfile}", fingerprint: true
|
|
copyToNightlies("${logfile}", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
|
|
}
|
|
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
|
|
dir("build") {
|
|
sh """
|
|
mkdir -p test/output/${cell.step}
|
|
find test/output -type f -name TEST*.xml -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
|
|
find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
|
|
find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
|
|
"""
|
|
junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
|
|
sh "find test/output -type f -name *.xml -exec sh -c 'xz -f {} &' ';' ; wait "
|
|
archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz", fingerprint: true
|
|
copyToNightlies("test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
|
|
}
|
|
cleanAgent(cell.step)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
def fetchSource(stage, arch, jdk) {
|
|
cleanAgent(stage)
|
|
if ("jar" == stage) {
|
|
checkout changelog: false, scm: scmGit(branches: [[name: params.branch ?: 'trunk']], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
|
|
sh "mkdir -p build/stage-logs"
|
|
} else {
|
|
unstash name: "${arch}_${jdk}"
|
|
}
|
|
}
|
|
|
|
def fetchDTestsSource(command, script_vars) {
|
|
if (command.containsKey('python-dtest')) {
|
|
checkout changelog: false, poll: false, scm: scmGit(branches: [[name: params.dtest_branch ?: 'trunk']], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true), [$class: 'RelativeTargetDirectory', relativeTargetDir: "${WORKSPACE}/build/cassandra-dtest"]], userRemoteConfigs: [[url: params.dtest_repository]])
|
|
sh "test -f build/cassandra-dtest/requirements.txt || { echo 'Invalid cassandra-dtest fork/branch'; exit 1; }"
|
|
return "${script_vars} cassandra_dtest_dir='${WORKSPACE}/build/cassandra-dtest'"
|
|
}
|
|
return script_vars
|
|
}
|
|
|
|
def buildJVMDTestJars(cell, script_vars, logfile) {
|
|
if (cell.step.startsWith("jvm-dtest-upgrade")) {
|
|
try {
|
|
unstash name: "jvm_dtests_${cell.arch}_${cell.jdk}"
|
|
} catch (error) {
|
|
sh label: "RUNNING build_dtest_jars...", script: "${script_vars} .build/docker/run-tests.sh build_dtest_jars ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )"
|
|
stash name: "jvm_dtests_${cell.arch}_${cell.jdk}", includes: '**/dtest*.jar'
|
|
}
|
|
}
|
|
}
|
|
|
|
def fetchDockerImages(dockerfiles) {
|
|
// prefetch, from apache jfrog, reduces risking dockerhub pull rate limits
|
|
// also prefetch alpine:latest as its used as a utility in the scripts
|
|
def dockerfilesVar = dockerfiles.join(' ')
|
|
sh """#!/bin/bash
|
|
for dockerfile in ${dockerfilesVar} ; do
|
|
image_tag="\$(md5sum .build/docker/\${dockerfile}.docker | cut -d' ' -f1)"
|
|
image_name="apache/cassandra-\${dockerfile}:\${image_tag}"
|
|
if ! ( [[ "" != "\$(docker images -q \${image_name} 2>/dev/null)" ]] ) ; then
|
|
docker pull -q apache.jfrog.io/cassan-docker/\${image_name} &
|
|
fi
|
|
done
|
|
docker pull -q apache.jfrog.io/cassan-docker/alpine:3.19.1 &
|
|
wait
|
|
"""
|
|
}
|
|
|
|
def getNodeLabel(command, cell) {
|
|
echo "using node label: cassandra-${cell.arch}-${command.size}"
|
|
return "cassandra-${cell.arch}-${command.size}"
|
|
}
|
|
|
|
def copyToNightlies(sourceFiles, remoteDirectory='') {
|
|
if (isCanonical() && sourceFiles?.trim()) {
|
|
def remotePath = remoteDirectory.startsWith("cassandra/") ? "${remoteDirectory}" : "cassandra/${JOB_NAME}/${BUILD_NUMBER}/${remoteDirectory}"
|
|
def attempt = 1
|
|
retry(9) {
|
|
if (attempt > 1) { sleep(60 * attempt) }
|
|
sshPublisher(
|
|
continueOnError: true, failOnError: false,
|
|
publishers: [
|
|
sshPublisherDesc(
|
|
configName: "Nightlies",
|
|
transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ]
|
|
)
|
|
])
|
|
}
|
|
echo "archived to https://nightlies.apache.org/${remotePath}"
|
|
}
|
|
}
|
|
|
|
def cleanAgent(job_name) {
|
|
sh "hostname"
|
|
if (isCanonical()) {
|
|
def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/"
|
|
cleanAgentDocker(job_name, agentScriptsUrl)
|
|
logAgentInfo(job_name, agentScriptsUrl)
|
|
}
|
|
cleanWs()
|
|
}
|
|
|
|
def cleanAgentDocker(job_name, agentScriptsUrl) {
|
|
// we don't expect any build to have been running for longer than maxBuildHours
|
|
def maxBuildHours = 12
|
|
echo "Pruning docker for '${job_name}' on ${NODE_NAME}…" ;
|
|
sh """#!/bin/bash
|
|
set +e
|
|
wget -q ${agentScriptsUrl}/docker_image_pruner.py
|
|
wget -q ${agentScriptsUrl}/docker_agent_cleaner.sh
|
|
bash docker_agent_cleaner.sh ${maxBuildHours}
|
|
"""
|
|
}
|
|
|
|
def logAgentInfo(job_name, agentScriptsUrl) {
|
|
sh """#!/bin/bash
|
|
set +e -o pipefail
|
|
wget -q ${agentScriptsUrl}/agent_report.sh
|
|
bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt
|
|
"""
|
|
copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/")
|
|
}
|
|
|
|
/////////////////////////////////////////
|
|
////// scripting support for summary ////
|
|
/////////////////////////////////////////
|
|
|
|
def generateTestReports() {
|
|
node("cassandra-medium") {
|
|
cleanAgent("generateTestReports")
|
|
checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
|
|
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_generateTestReports.log.xz"
|
|
sh "mkdir -p build/stage-logs"
|
|
def teeSuffix = "2>&1 | tee >( xz -c > build/${logfile} )"
|
|
def script_vars = "#!/bin/bash -x \n "
|
|
if (isCanonical()) {
|
|
// copyArtifacts takes >4hrs, hack with manual download
|
|
sh """${script_vars}
|
|
( mkdir -p build/test
|
|
wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip
|
|
unzip -x -d build/test -q output.zip ) ${teeSuffix}
|
|
"""
|
|
} else {
|
|
copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
|
|
}
|
|
if (fileExists('build/test/output')) {
|
|
// merge splits for each target's test report, other axes are kept separate
|
|
// TODO parallelised for loop
|
|
// TODO results_details.tar.xz needs to include all logs for failed tests
|
|
sh """${script_vars} (
|
|
find build/test/output -name *.xml.xz -exec sh -c 'xz -f --decompress {} &' ';' ; wait
|
|
|
|
for target in \$(ls build/test/output/) ; do
|
|
if test -d build/test/output/\${target} ; then
|
|
mkdir -p build/test/reports/\${target}
|
|
echo "Report for \${target} (\$(find build/test/output/\${target} -name '*.xml' | wc -l) test files)"
|
|
CASSANDRA_DOCKER_ANT_OPTS="-Dbuild.test.output.dir=build/test/output/\${target} -Dbuild.test.report.dir=build/test/reports/\${target}"
|
|
export CASSANDRA_DOCKER_ANT_OPTS
|
|
.build/docker/_docker_run.sh bullseye-build.docker ci/generate-test-report.sh
|
|
fi
|
|
done
|
|
|
|
.build/docker/_docker_run.sh bullseye-build.docker ci/generate-ci-summary.sh || echo "failed generate-ci-summary.sh"
|
|
|
|
tar -cf build/results_details.tar -C build/test/ reports
|
|
xz -8f build/results_details.tar ) ${teeSuffix}
|
|
"""
|
|
|
|
dir('build/') {
|
|
archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true
|
|
copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile}')
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
def sendNotifications() {
|
|
if (isPostCommit() && isCanonical()) {
|
|
// the following is expected only to work on ci-cassandra.apache.org
|
|
def changes = '?'
|
|
try {
|
|
script {
|
|
changes = formatChangeLogChanges(currentBuild.changeSets)
|
|
echo "changes: ${changes}"
|
|
}
|
|
slackSend channel: '#cassandra-builds', message: ":apache: <${BUILD_URL}|${currentBuild.fullDisplayName}> completed: ${currentBuild.result}. <https://github.com/apache/cassandra/commit/${GIT_COMMIT}|${GIT_COMMIT}>\n${changes}"
|
|
emailext to: 'builds@cassandra.apache.org', subject: "Build complete: ${currentBuild.fullDisplayName} [${currentBuild.result}] ${GIT_COMMIT}", presendScript: 'msg.removeHeader("In-Reply-To"); msg.removeHeader("References")', body: emailContent()
|
|
} catch (Exception ex) {
|
|
echo 'failed to send notifications ' + ex.toString()
|
|
}
|
|
}
|
|
}
|
|
|
|
def formatChangeLogChanges(changeLogSets) {
|
|
def result = ''
|
|
for (int i = 0; i < changeLogSets.size(); i++) {
|
|
def entries = changeLogSets[i].items
|
|
for (int j = 0; j < entries.length; j++) {
|
|
def entry = entries[j]
|
|
result = result + "${entry.commitId} by ${entry.author} on ${new Date(entry.timestamp)}: ${entry.msg}\n"
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
def emailContent() {
|
|
return '''
|
|
-------------------------------------------------------------------------------
|
|
Build ${ENV,var="JOB_NAME"} #${BUILD_NUMBER} ${BUILD_STATUS}
|
|
URL: ${BUILD_URL}
|
|
-------------------------------------------------------------------------------
|
|
Changes:
|
|
${CHANGES}
|
|
-------------------------------------------------------------------------------
|
|
Failed Tests:
|
|
${FAILED_TESTS,maxTests=500,showMessage=false,showStack=false}
|
|
-------------------------------------------------------------------------------
|
|
For complete test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/
|
|
'''
|
|
}
|