diff --git a/.build/build-bench.xml b/.build/build-bench.xml index b10fc64425..5cd171b8a2 100644 --- a/.build/build-bench.xml +++ b/.build/build-bench.xml @@ -20,6 +20,11 @@ xmlns:if="ant:if" xmlns:unless="ant:unless"> + + + + + @@ -81,6 +86,7 @@ + @@ -103,7 +109,7 @@ - + @@ -111,10 +117,9 @@ - - + - + diff --git a/.build/run-tests.sh b/.build/run-tests.sh index 68ecbff7f6..7f91757e87 100755 --- a/.build/run-tests.sh +++ b/.build/run-tests.sh @@ -281,16 +281,85 @@ _run_testlist() { [ "${_test_iterations}" -eq 1 ] || printf "––––\nfailure rate: ${failures}/${_test_iterations}\n" } +_list_microbench_tests() { + # Extract blacklist from build-bench.xml property (see CASSANDRA-18873) + local blacklist_pattern=$(grep 'name="microbench.exclude.pattern"' .build/build-bench.xml | sed -n 's/.*value="\([^"]*\)".*/\1/p') + + # Find all *Bench.java files, strip prefix, sort, and filter out blacklisted ones + find "test/microbench" -name '*Bench.java' | \ + sed "s;^test/microbench/;;g" | \ + sort | \ + grep -vE "(${blacklist_pattern})\.java$" +} + +_run_microbench() { + local _target=$1 + local _test_name_regexp=$2 + local _split_chunk=$3 + local testlist="" + + # Assert no *Test.java files exist under test/microbench + # uncomment once CachingBenchTest and GcCompactionBenchTest are rewritten to JMH benchmarks + #_list_tests "microbench" | grep -q 'Test\.java$' && error 1 "Found *Test.java files under test/microbench, these should be moved to test/unit" + + # Build test list from either regexp or split + if [ -n "${_test_name_regexp}" ]; then + echo "Running tests: ${_test_name_regexp}" + # test regexp can come in csv + for i in ${_test_name_regexp//,/ }; do + [ -n "${testlist}" ] && testlist="${testlist}"$'\n' + testlist="${testlist}$( _list_microbench_tests | _split_tests "${i}")" + done + [[ -z "${testlist}" ]] && error 1 "No tests found in test name regexp: ${_test_name_regexp}" + else + [ -n "${_split_chunk}" ] || { error 1 "Neither name regexp or split chunk defined"; } + echo "Running split: ${_split_chunk}" + testlist="$( _list_microbench_tests | _split_tests "${_split_chunk}")" + if [[ -z "${testlist}" ]]; then + echo "No microbench tests in split ${_split_chunk}, skipping" + return 0 + fi + fi + + # Convert file paths to the JMH classname pattern + local benchmark_pattern=$(echo "${testlist}" | sed 's/\.java$//g' | sed 's|^org/apache/cassandra/test/microbench/||g' | sed 's/\//./g' | tr '\n' '|' | sed 's/|$//') + echo "Running benchmarks: ${benchmark_pattern}" + + # override build.test.output.dir, adding jdk and arch to output path for report separation + local -r java_version="$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')" + local -r arch="$(uname -m)" + local -r output_dir="${DIST_DIR}/test/output/${_target}/jdk${java_version}/${arch}/${_split_chunk//\//_}" + + ant $_target ${ANT_TEST_OPTS} -Dbuild.test.output.dir=${output_dir} -Dbenchmark.name="${benchmark_pattern}" -Dmaven.test.failure.ignore=true + + # Post-process jmh-result.json to add jdk and arch parameters + local jmh_result="${output_dir}/jmh-result.json" + if [ -f "${jmh_result}" ]; then + python3 -c " +import json,sys +with open('${jmh_result}','r') as f: + data=json.load(f) +for r in (data if isinstance(data,list) else [data]): + if 'params' not in r: + r['params']={} + r['params']['jdk']='${java_version}' + r['params']['arch']='${arch}' +with open('${jmh_result}','w') as f: + json.dump(data,f) +" + fi +} + _main() { # parameters local -r target="${test_target/-repeat/}" - local -r split_chunk="${chunk:-'1/1'}" # Chunks formatted as "K/N" for the Kth chunk of N chunks + local -r split_chunk="${chunk:-1/1}" # Chunks formatted as "K/N" for the Kth chunk of N chunks # check split_chunk is compatible with target (if not a regexp) if [[ "${_split_chunk}" =~ ^\d+/\d+$ ]] && [[ "1/1" != "${split_chunk}" ]] ; then case ${target} in - "stress-test" | "fqltool-test" | "microbench" | "cqlsh-test" | "simulator-dtest") - error 1 "Target ${target} does not suport splits." + "stress-test" | "fqltool-test" | "cqlsh-test" | "simulator-dtest") + error 1 "Target ${target} does not support splits." ;; *) ;; @@ -349,8 +418,7 @@ _main() { ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}" ;; "microbench" | "microbench-test") - [[ "x${test_name_regexp}" != "x" ]] && test_name_regexp="-Dbenchmark.name=${test_name_regexp}" - ant $target ${ANT_TEST_OPTS} ${test_name_regexp} -Dmaven.test.failure.ignore=true + _run_microbench "$target" "${test_name_regexp}" "${split_chunk}" ;; "test") _run_testlist "unit" "testclasslist" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}" diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index ea23a4bbfd..b4d8826d11 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -213,9 +213,9 @@ def tasks() { 'dtest-upgrade-novnode': [splits: 128, size: 'large'], 'dtest-upgrade-large': [splits: 32, size: 'large'], 'dtest-upgrade-large-novnode': [splits: 32, size: 'large'], - 'microbench-test': [splits: 1, size: 'large'], + 'microbench-test': [splits: 4, size: 'large', timeout_hours: 2], // performance tests need 'cassandra-*large-dedicated' nodes - 'microbench': [splits: 1, size: 'large', timeout_hours: 6, benchmark: true], + 'microbench': [splits: 4, size: 'large', timeout_hours: 6, benchmark: true], ] testSteps.each() { it.value.put('type', 'test') @@ -470,8 +470,8 @@ def test(command, cell) { find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l """ - archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true - copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) + archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true + copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) } } finally { cleanAgent(cell.step) @@ -619,9 +619,10 @@ def generateTestReports() { unzip -x -d build/test -q output.zip ) ${teeSuffix} """ } else { - copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true + copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true } - if (fileExists('build/test/output')) { + // merge and summarise test reports + if (fileExists('build/test/output') && sh(script: 'test -n "$(find build/test/output -type f -name "*.xml.xz" -print -quit)"', returnStatus: true) == 0) { // merge splits for each target's test report, other axes are kept separate // TODO parallelised for loop // TODO results_details.tar.xz needs to include all logs for failed tests @@ -647,11 +648,31 @@ def generateTestReports() { dir('build/') { archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true - copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile},test/jmh-result.json') + copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile}') } } - if (fileExists('build/test/jmh-result.json')) { - jmhReport('build/test/jmh-result.json') + processJmhReports() + } +} + +def processJmhReports() { + if (fileExists('build/test/output') && sh(script: 'test -n "$(find build/test/output -type f -name jmh-result.json -print -quit)"', returnStatus: true) == 0) { + sh ''' python3 -c " +import glob,json +m=[] +for f in glob.glob('build/test/output/**/jmh-result.json',recursive=True): + d=json.load(open(f)) + m.extend(d if isinstance(d,list) else [d]) +o=open('build/test/output/combined-result.json','w') +json.dump(m,o) +o.close() +print(f'combined {len(m)} results') +" + ''' + jmhReport('build/test/output/combined-result.json') + dir('build/') { + archiveArtifacts artifacts: "test/output/combined-result.json", fingerprint: true + copyToNightlies('test/output/combined-result.json') } } } diff --git a/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java b/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java index 3a0f09fc2a..40d12d43b0 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java +++ b/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java @@ -54,6 +54,8 @@ import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.assertj.core.api.Assertions.assertThat; +// FIXME – rewrite to jmh bench classes +// https://issues.apache.org/jira/browse/CASSANDRA-17964?focusedCommentId=17680480&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-17680480 public class CachingBenchTest extends CQLTester { private static final String STRATEGY = "LeveledCompactionStrategy"; diff --git a/test/microbench/org/apache/cassandra/test/microbench/GcCompactionBenchTest.java b/test/microbench/org/apache/cassandra/test/microbench/GcCompactionBenchTest.java index cd905e10d3..f96775f851 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/GcCompactionBenchTest.java +++ b/test/microbench/org/apache/cassandra/test/microbench/GcCompactionBenchTest.java @@ -52,6 +52,8 @@ import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.Clock.Global.nanoTime; +// FIXME – rewrite to jmh bench classes +// https://issues.apache.org/jira/browse/CASSANDRA-17964?focusedCommentId=17680480&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-17680480 public class GcCompactionBenchTest extends CQLTester { private static final String SIZE_TIERED_STRATEGY = "SizeTieredCompactionStrategy', 'min_sstable_size' : '0";