natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

In jenkins retry each stage build in the pipeline 

attempts to alleviate the false-negatives from CI infrastructure faults (disks, network, etc)

 patch by Mick Semb Wever; reviewed by Brandon Williams for CASSANDRA-17664
This commit is contained in:
Mick Semb Wever 2022-05-26 14:06:47 +02:00
parent 9c0f8902e8
commit 136c8c0f65
No known key found for this signature in database
GPG Key ID: E91335D77E3E87CB
1 changed files with 107 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
.jenkins/Jenkinsfile vendored
View File

@ -29,37 +29,38 @@ pipeline {
stage('Init') { stage('Init') {
steps { steps {
cleanWs() cleanWs()
script {
currentBuild.result='SUCCESS'
}
} }
} }
stage('Build') { stage('Build') {
steps { steps {
build job: "${env.JOB_NAME}-artifacts" script {
def attempt = 1
retry(2) {
if (attempt > 1) {
sleep(60 * attempt)
}
attempt = attempt + 1
build job: "${env.JOB_NAME}-artifacts"
}
}
} }
} }
stage('Test') { stage('Test') {
parallel { parallel {
stage('JVM DTests') {
steps {
script {
jvm_dtest = build job: "${env.JOB_NAME}-jvm-dtest", propagate: false
if (jvm_dtest.result != 'SUCCESS') unstable('jvm-dtest failures')
if (jvm_dtest.result == 'FAILURE') currentBuild.result='FAILURE'
}
}
post {
always {
warnError('missing test xml files') {
script {
copyTestResults('jvm-dtest', jvm_dtest.getNumber())
}
}
}
}
}
stage('units') { stage('units') {
steps { steps {
script { script {
test = build job: "${env.JOB_NAME}-test", propagate: false def attempt = 1
retry(2) {
if (attempt > 1) {
sleep(60 * attempt)
}
attempt = attempt + 1
test = build job: "${env.JOB_NAME}-test", propagate: false
}
if (test.result != 'SUCCESS') unstable('unit test failures') if (test.result != 'SUCCESS') unstable('unit test failures')
if (test.result == 'FAILURE') currentBuild.result='FAILURE' if (test.result == 'FAILURE') currentBuild.result='FAILURE'
} }
@ -77,7 +78,14 @@ pipeline {
stage('long units') { stage('long units') {
steps { steps {
script { script {
long_test = build job: "${env.JOB_NAME}-long-test", propagate: false def attempt = 1
retry(2) {
if (attempt > 1) {
sleep(60 * attempt)
}
attempt = attempt + 1
long_test = build job: "${env.JOB_NAME}-long-test", propagate: false
}
if (long_test.result != 'SUCCESS') unstable('long unit test failures') if (long_test.result != 'SUCCESS') unstable('long unit test failures')
if (long_test.result == 'FAILURE') currentBuild.result='FAILURE' if (long_test.result == 'FAILURE') currentBuild.result='FAILURE'
} }
@ -95,7 +103,14 @@ pipeline {
stage('burn') { stage('burn') {
steps { steps {
script { script {
burn = build job: "${env.JOB_NAME}-test-burn", propagate: false def attempt = 1
retry(2) {
if (attempt > 1) {
sleep(60 * attempt)
}
attempt = attempt + 1
burn = build job: "${env.JOB_NAME}-test-burn", propagate: false
}
if (burn.result != 'SUCCESS') unstable('burn test failures') if (burn.result != 'SUCCESS') unstable('burn test failures')
if (burn.result == 'FAILURE') currentBuild.result='FAILURE' if (burn.result == 'FAILURE') currentBuild.result='FAILURE'
} }
@ -113,7 +128,14 @@ pipeline {
stage('compression') { stage('compression') {
steps { steps {
script { script {
compression = build job: "${env.JOB_NAME}-test-compression", propagate: false def attempt = 1
retry(2) {
if (attempt > 1) {
sleep(60 * attempt)
}
attempt = attempt + 1
compression = build job: "${env.JOB_NAME}-test-compression", propagate: false
}
if (compression.result != 'SUCCESS') unstable('compression failures') if (compression.result != 'SUCCESS') unstable('compression failures')
if (compression.result == 'FAILURE') currentBuild.result='FAILURE' if (compression.result == 'FAILURE') currentBuild.result='FAILURE'
} }
@ -132,10 +154,42 @@ pipeline {
} }
stage('Distributed Test') { stage('Distributed Test') {
parallel { parallel {
stage('JVM DTests') {
steps {
script {
def attempt = 1
retry(2) {
if (attempt > 1) {
sleep(60 * attempt)
}
attempt = attempt + 1
jvm_dtest = build job: "${env.JOB_NAME}-jvm-dtest", propagate: false
}
if (jvm_dtest.result != 'SUCCESS') unstable('jvm-dtest failures')
if (jvm_dtest.result == 'FAILURE') currentBuild.result='FAILURE'
}
}
post {
always {
warnError('missing test xml files') {
script {
copyTestResults('jvm-dtest', jvm_dtest.getNumber())
}
}
}
}
}
stage('dtest') { stage('dtest') {
steps { steps {
script { script {
dtest = build job: "${env.JOB_NAME}-dtest", propagate: false def attempt = 1
retry(2) {
if (attempt > 1) {
sleep(60 * attempt)
}
attempt = attempt + 1
dtest = build job: "${env.JOB_NAME}-dtest", propagate: false
}
if (dtest.result != 'SUCCESS') unstable('dtest failures') if (dtest.result != 'SUCCESS') unstable('dtest failures')
if (dtest.result == 'FAILURE') currentBuild.result='FAILURE' if (dtest.result == 'FAILURE') currentBuild.result='FAILURE'
} }
@ -153,7 +207,14 @@ pipeline {
stage('dtest-large') { stage('dtest-large') {
steps { steps {
script { script {
dtest_large = build job: "${env.JOB_NAME}-dtest-large", propagate: false def attempt = 1
retry(2) {
if (attempt > 1) {
sleep(60 * attempt)
}
attempt = attempt + 1
dtest_large = build job: "${env.JOB_NAME}-dtest-large", propagate: false
}
if (dtest_large.result != 'SUCCESS') unstable('dtest-large failures') if (dtest_large.result != 'SUCCESS') unstable('dtest-large failures')
if (dtest_large.result == 'FAILURE') currentBuild.result='FAILURE' if (dtest_large.result == 'FAILURE') currentBuild.result='FAILURE'
} }
@ -171,7 +232,14 @@ pipeline {
stage('dtest-novnode') { stage('dtest-novnode') {
steps { steps {
script { script {
dtest_novnode = build job: "${env.JOB_NAME}-dtest-novnode", propagate: false def attempt = 1
retry(2) {
if (attempt > 1) {
sleep(60 * attempt)
}
attempt = attempt + 1
dtest_novnode = build job: "${env.JOB_NAME}-dtest-novnode", propagate: false
}
if (dtest_novnode.result != 'SUCCESS') unstable('dtest-novnode failures') if (dtest_novnode.result != 'SUCCESS') unstable('dtest-novnode failures')
if (dtest_novnode.result == 'FAILURE') currentBuild.result='FAILURE' if (dtest_novnode.result == 'FAILURE') currentBuild.result='FAILURE'
} }
@ -189,7 +257,14 @@ pipeline {
stage('dtest-offheap') { stage('dtest-offheap') {
steps { steps {
script { script {
dtest_offheap = build job: "${env.JOB_NAME}-dtest-offheap", propagate: false def attempt = 1
retry(2) {
if (attempt > 1) {
sleep(60 * attempt)
}
attempt = attempt + 1
dtest_offheap = build job: "${env.JOB_NAME}-dtest-offheap", propagate: false
}
if (dtest_offheap.result != 'SUCCESS') unstable('dtest-offheap failures') if (dtest_offheap.result != 'SUCCESS') unstable('dtest-offheap failures')
if (dtest_offheap.result == 'FAILURE') currentBuild.result='FAILURE' if (dtest_offheap.result == 'FAILURE') currentBuild.result='FAILURE'
} }
@ -235,14 +310,15 @@ ${FAILED_TESTS,maxTests=500,showMessage=false,showStack=false}
For complete test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/ For complete test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/
''' '''
} }
sh "echo \"cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`\" > builds.head" sh "echo \"summary) cassandra-builds: `git -C cassandra-builds log -1 --pretty=format:'%H %an %ad %s'`\" > builds.head"
sh "find . -type f -name \\*.head -exec cat {} \\;" sh "./cassandra-builds/jenkins-dsl/print-shas.sh"
sh "xz TESTS-TestSuites.xml" sh "xz TESTS-TestSuites.xml"
sh "echo \"For test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/\"" sh "wget --retry-connrefused --waitretry=1 \"\${BUILD_URL}/timestamps/?time=HH:mm:ss&timeZone=UTC&appendLog\" -qO - > console.log || echo wget failed"
sh "xz console.log"
} }
post { post {
always { always {
sshPublisher(publishers: [sshPublisherDesc(configName: 'Nightlies', transfers: [sshTransfer(remoteDirectory: 'cassandra/${JOB_NAME}/${BUILD_NUMBER}/', sourceFiles: 'TESTS-TestSuites.xml.xz')])]) sshPublisher(publishers: [sshPublisherDesc(configName: 'Nightlies', transfers: [sshTransfer(remoteDirectory: 'cassandra/${JOB_NAME}/${BUILD_NUMBER}/', sourceFiles: 'console.log.xz,TESTS-TestSuites.xml.xz')])])
} }
} }
} }