mirror of https://github.com/apache/cassandra
Merge branch 'cassandra-5.0' into trunk
* cassandra-5.0: Improve jenkinsfile parse perf, and fix parallel xz test result file compressing
This commit is contained in:
Mick Semb Wever
commit
da31410ac6
|
|
@ -35,7 +35,7 @@
|
|||
// - cassandra-amd64-medium : 3 cpu, 5GB ram
|
||||
// - cassandra-medium : 3 cpu, 5GB ram (alias for above but for any arch)
|
||||
// - cassandra-amd64-large : 7 cpu, 16GB ram
|
||||
// - cassandra-large : 7 cpu, 16GB ram
|
||||
// - cassandra-large : 7 cpu, 16GB ram
|
||||
//
|
||||
// When running builds parameterised to other architectures the corresponding labels are expected.
|
||||
// For example 'arm64' requires the labels: cassandra-arm64-small, cassandra-arm64-medium, cassandra-arm64-large.
|
||||
|
|
@ -56,6 +56,12 @@
|
|||
// `curl -X POST -F "jenkinsfile=<.jenkins/Jenkinsfile" https://ci-cassandra.apache.org/pipeline-model-converter/validate`
|
||||
//
|
||||
|
||||
/** CONSTANTS for both the pipeline and scripting **/
|
||||
import groovy.transform.Field
|
||||
@Field List<String> archsSupported = ["amd64", "arm64"]
|
||||
@Field List<String> pythonsSupported = ["3.8", "3.11"]
|
||||
@Field String pythonDefault = "3.8"
|
||||
/** CONSTANTS end **********************************/
|
||||
|
||||
pipeline {
|
||||
agent { label 'cassandra-small' }
|
||||
|
|
@ -70,13 +76,22 @@ pipeline {
|
|||
choice(name: 'profile', choices: pipelineProfileNames(params.profile ?: ''), description: 'Pick a pipeline profile.')
|
||||
string(name: 'profile_custom_regexp', defaultValue: params.profile_custom_regexp ?: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*')
|
||||
|
||||
choice(name: 'architecture', choices: archsSupported() + "all", description: 'Pick architecture. The ARM64 is disabled by default at the moment.')
|
||||
choice(name: 'architecture', choices: archsSupported + "all", description: 'Pick architecture. The ARM64 is disabled by default at the moment.')
|
||||
string(name: 'jdk', defaultValue: params.jdk ?: '', description: 'Restrict JDK versions. (e.g. "11", "17", etc)')
|
||||
|
||||
string(name: 'dtest_repository', defaultValue: params.dtest_repository ?: 'https://github.com/apache/cassandra-dtest', description: 'Cassandra DTest Repository')
|
||||
string(name: 'dtest_branch', defaultValue: params.dtest_branch ?: 'trunk', description: 'DTest Branch')
|
||||
}
|
||||
stages {
|
||||
stage('init') {
|
||||
steps {
|
||||
script {
|
||||
// this helps assure folk their parameters are correct and will be used (despite the earlier output about the configured job coordinates)
|
||||
echo "Printing parameters used for this build"
|
||||
["Repository: ${params.repository}", "Branch: ${params.branch}", "Profile: ${params.profile}", "Custom Profile Regexp: ${params.profile_custom_regexp}", "Architecture: ${params.architecture}", "JDK: ${params.jdk}", "DTest Repository: ${params.dtest_repository}", "DTest Branch: ${params.dtest_branch}"].each { println it }
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('jar') {
|
||||
// the jar stage executes only the 'jar' build step, via the build(…) function
|
||||
// the results of these (per jdk, per arch) are then stashed and used for every other build and test step
|
||||
|
|
@ -120,10 +135,7 @@ pipeline {
|
|||
//// scripting support ////
|
||||
///////////////////////////
|
||||
|
||||
def archsSupported() { return ["amd64", "arm64"] }
|
||||
def pythonsSupported() { return ["3.8", "3.11"] }
|
||||
def pythonDefault() { return "3.8" }
|
||||
|
||||
@NonCPS
|
||||
def pipelineProfiles() {
|
||||
return [
|
||||
'packaging': ['artifacts', 'lint', 'debian', 'redhat'],
|
||||
|
|
@ -135,6 +147,7 @@ def pipelineProfiles() {
|
|||
]
|
||||
}
|
||||
|
||||
@NonCPS
|
||||
def pipelineProfileNames(putFirst) {
|
||||
set = pipelineProfiles().keySet() as List
|
||||
set = set - putFirst
|
||||
|
|
@ -142,7 +155,11 @@ def pipelineProfileNames(putFirst) {
|
|||
return set
|
||||
}
|
||||
|
||||
@Field Map cachedTasks = null
|
||||
|
||||
def tasks() {
|
||||
if (null != cachedTasks) return cachedTasks
|
||||
|
||||
// Steps config
|
||||
def buildSteps = [
|
||||
'jar': [script: 'build-jars.sh', toCopy: null],
|
||||
|
|
@ -201,23 +218,30 @@ def tasks() {
|
|||
|
||||
def stepsMap = buildSteps + testSteps
|
||||
|
||||
// find the default JDK and the supported JDKs defined in the build.xml
|
||||
def build_xml = readFile(file: 'build.xml')
|
||||
def javaVersionDefaultMatch = (build_xml =~ /property\s*name="java\.default"\s*value="([^"]*)"/)
|
||||
assert javaVersionDefaultMatch, "java.default property not found in build.xml"
|
||||
def javaVersionDefault = javaVersionDefaultMatch[0][1]
|
||||
def javaVersionsSupportedMatch = (build_xml =~ /property\s*name="java\.supported"\s*value="([^"]*)"/)
|
||||
assert javaVersionsSupportedMatch, "java.supported property not found in build.xml"
|
||||
def javaVersionsSupported = javaVersionsSupportedMatch[0][1].split(',') as List
|
||||
|
||||
// define matrix axes
|
||||
def Map matrix_axes = [
|
||||
arch: archsSupported(),
|
||||
jdk: javaVersionsSupported(),
|
||||
python: pythonsSupported(),
|
||||
arch: archsSupported,
|
||||
jdk: javaVersionsSupported,
|
||||
python: pythonsSupported,
|
||||
cython: ['yes', 'no'],
|
||||
step: stepsMap.keySet(),
|
||||
split: (1..testSteps.values().splits.max()).toList()
|
||||
]
|
||||
|
||||
def javaVersionDefault = javaVersionDefault()
|
||||
|
||||
def List _axes = getMatrixAxes(matrix_axes).findAll { axis ->
|
||||
(isArchEnabled(axis['arch'])) && // skip disabled archs
|
||||
(isJdkEnabled(axis['jdk'])) && // skip disabled jdks
|
||||
(isStageEnabled(axis['step'])) && // skip disabled steps
|
||||
!(axis['python'] != pythonDefault() && 'cqlsh-test' != axis['step']) && // Use only python 3.8 for all tests but cqlsh-test
|
||||
!(axis['python'] != pythonDefault && 'cqlsh-test' != axis['step']) && // Use only python 3.8 for all tests but cqlsh-test
|
||||
!(axis['cython'] != 'no' && 'cqlsh-test' != axis['step']) && // cython only for cqlsh-test, disable for others
|
||||
!(axis['jdk'] != javaVersionDefault && ('cqlsh-test' == axis['step'] || 'simulator-dtest' == axis['step'] || axis['step'].contains('dtest-upgrade'))) && // run cqlsh-test, simulator-dtest, *dtest-upgrade only with jdk11
|
||||
// Disable splits for all but proper stages
|
||||
|
|
@ -239,7 +263,7 @@ def tasks() {
|
|||
}
|
||||
}
|
||||
|
||||
return tasks
|
||||
return cachedTasks = tasks
|
||||
}
|
||||
|
||||
@NonCPS
|
||||
|
|
@ -265,34 +289,14 @@ def getStepName(cell, command) {
|
|||
|
||||
def getJarTasks() {
|
||||
Map jars = tasks()['jars']
|
||||
assertJarTasks(jars)
|
||||
assert jars.size() > 1, "Nothing to build. Check parameters: jdk ${params.jdk}, arch ${params.architecture}"
|
||||
return jars
|
||||
}
|
||||
|
||||
def assertJarTasks(jars) {
|
||||
if (jars.size() < 2) {
|
||||
error("Nothing to build. Check parameters: jdk ${params.jdk} (${javaVersionsSupported()}), arch ${params.architecture} (${archsSupported()})")
|
||||
}
|
||||
}
|
||||
|
||||
def hasNonJarTasks() {
|
||||
return tasks()['tests'].size() > 1
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the default JDK defined by build.xml
|
||||
**/
|
||||
def javaVersionDefault() {
|
||||
sh (returnStdout: true, script: 'grep \'property\\s*name=\"java.default\"\' build.xml | sed -ne \'s/.*value=\"\\([^\"]*\\)\".*/\\1/p\'').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the supported JDKs defined by build.xml
|
||||
**/
|
||||
def javaVersionsSupported() {
|
||||
sh (returnStdout: true, script: 'grep \'property\\s*name=\"java.supported\"\' build.xml | sed -ne \'s/.*value=\"\\([^\"]*\\)\".*/\\1/p\'').trim().split(',')
|
||||
}
|
||||
|
||||
/**
|
||||
* Is this a post-commit build (or a pre-commit build)
|
||||
**/
|
||||
|
|
@ -413,12 +417,15 @@ def test(command, cell) {
|
|||
dir("build") {
|
||||
sh """
|
||||
mkdir -p test/output/${cell.step}
|
||||
find test/output -type f -name TEST*.xml -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
|
||||
find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
|
||||
find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
|
||||
find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
|
||||
"""
|
||||
junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
|
||||
sh "find test/output -type f -name *.xml -exec sh -c 'xz -f {} &' ';' ; wait ; find test/output -type f -name *.xml.xz | wc -l"
|
||||
sh """
|
||||
find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
|
||||
echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
|
||||
"""
|
||||
archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz", fingerprint: true
|
||||
copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
|
||||
}
|
||||
|
|
@ -558,8 +565,8 @@ def generateTestReports() {
|
|||
// TODO parallelised for loop
|
||||
// TODO results_details.tar.xz needs to include all logs for failed tests
|
||||
sh """${script_vars} (
|
||||
find build/test/output -type f -name *.xml.xz | wc -l
|
||||
find build/test/output -name *.xml.xz -exec sh -c 'xz -f --decompress {} &' ';' ; wait
|
||||
echo "test result files to decompress"; find build/test/output -type f -name "*.xml.xz" | wc -l
|
||||
find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f --decompress
|
||||
|
||||
for target in \$(ls build/test/output/) ; do
|
||||
if test -d build/test/output/\${target} ; then
|
||||
|
|
|
|||
Loading…
Reference in New Issue