Implement splits for microbench test type

Also distinguish benches by jdk version and host arch.

 patch by Mick Semb Wever; reviewed by Dmitry Konstantinov for CASSANDRA-21242

.build/build-bench.xml

@@ -20,6 +20,11 @@
          xmlns:if="ant:if" xmlns:unless="ant:unless">
 
     <property name="async-profiler.version" value="2.9"/>
+
+    <!-- TODO https://issues.apache.org/jira/browse/CASSANDRA-18873 -->
+    <!-- Blacklist of broken benchmarks -->
+    <property name="microbench.exclude.pattern" value="AtomicBTreePartitionUpdateBench|BTreeSearchIteratorBench|BTreeTransformBench|BTreeUpdateBench|FastThreadLocalBench|KeyLookupBench|MutationBench|(instance.*Bench)|ReadWriteBench|ZeroCopyStreamingBench"/>
+
     <condition property="async-profiler.suffix" value="linux-arm64">
         <and>
             <os arch="aarch64"/>
@@ -81,6 +86,7 @@
     <macrodef name="jmh">
         <element name="extra-args" optional="true"/>
         <sequential>
+            <mkdir dir="${build.test.output.dir}"/>
             <java classname="org.openjdk.jmh.Main" fork="true" failonerror="true" >
                 <classpath>
                     <path refid="cassandra.classpath.test"/>
@@ -103,7 +109,7 @@
                 <arg value="-rf"/>
                 <arg value="json"/>
                 <arg value="-rff"/>
-                <arg value="${build.test.dir}/jmh-result.json"/>
+                <arg value="${build.test.output.dir}/jmh-result.json"/>
                 <arg value="-v"/>
                 <arg value="EXTRA"/>
 
@@ -111,10 +117,9 @@
 
                 <extra-args/>
 
-                <!-- TODO https://issues.apache.org/jira/browse/CASSANDRA-18873 -->
-                <!-- The classes listed below are broken, and so currently ignored, see CASSANDRA-18873 -->
+                <!-- Exclude broken, see CASSANDRA-18873 -->
                 <arg value="-e" if:blank="${benchmark.name}"/>
-                <arg value="AtomicBTreePartitionUpdateBench|BTreeSearchIteratorBench|BTreeTransformBench|BTreeUpdateBench|FastThreadLocalBench|KeyLookupBench|MutationBench|(instance.*Bench)|ReadWriteBench|ZeroCopyStreamingBench" if:blank="${benchmark.name}"/>
+                <arg value="${microbench.exclude.pattern}" if:blank="${benchmark.name}"/>
 
                 <arg value=".*microbench.*${benchmark.name}"/>
             </java>

.build/run-tests.sh

@@ -281,16 +281,85 @@ _run_testlist() {
     [ "${_test_iterations}" -eq 1 ] || printf "––––\nfailure rate: ${failures}/${_test_iterations}\n"
 }
 
+_list_microbench_tests() {
+    # Extract blacklist from build-bench.xml property (see CASSANDRA-18873)
+    local blacklist_pattern=$(grep 'name="microbench.exclude.pattern"' .build/build-bench.xml | sed -n 's/.*value="\([^"]*\)".*/\1/p')
+
+    # Find all *Bench.java files, strip prefix, sort, and filter out blacklisted ones
+    find "test/microbench" -name '*Bench.java' | \
+        sed "s;^test/microbench/;;g" | \
+        sort | \
+        grep -vE "(${blacklist_pattern})\.java$"
+}
+
+_run_microbench() {
+    local _target=$1
+    local _test_name_regexp=$2
+    local _split_chunk=$3
+    local testlist=""
+
+    # Assert no *Test.java files exist under test/microbench
+    # uncomment once CachingBenchTest and GcCompactionBenchTest are rewritten to JMH benchmarks
+    #_list_tests "microbench" | grep -q 'Test\.java$' && error 1 "Found *Test.java files under test/microbench, these should be moved to test/unit"
+
+    # Build test list from either regexp or split
+    if [ -n "${_test_name_regexp}" ]; then
+      echo "Running tests: ${_test_name_regexp}"
+      # test regexp can come in csv
+      for i in ${_test_name_regexp//,/ }; do
+        [ -n "${testlist}" ] && testlist="${testlist}"$'\n'
+        testlist="${testlist}$( _list_microbench_tests | _split_tests "${i}")"
+      done
+      [[ -z "${testlist}" ]] && error 1 "No tests found in test name regexp: ${_test_name_regexp}"
+    else
+      [ -n "${_split_chunk}" ] || { error 1 "Neither name regexp or split chunk defined"; }
+      echo "Running split: ${_split_chunk}"
+      testlist="$( _list_microbench_tests | _split_tests "${_split_chunk}")"
+      if [[ -z "${testlist}" ]]; then
+        echo "No microbench tests in split ${_split_chunk}, skipping"
+        return 0
+      fi
+    fi
+
+    # Convert file paths to the JMH classname pattern
+    local benchmark_pattern=$(echo "${testlist}" | sed 's/\.java$//g' | sed 's|^org/apache/cassandra/test/microbench/||g' | sed 's/\//./g' | tr '\n' '|' | sed 's/|$//')
+    echo "Running benchmarks: ${benchmark_pattern}"
+
+    # override build.test.output.dir, adding jdk and arch to output path for report separation
+    local -r java_version="$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')"
+    local -r arch="$(uname -m)"
+    local -r output_dir="${DIST_DIR}/test/output/${_target}/jdk${java_version}/${arch}/${_split_chunk//\//_}"
+
+    ant $_target ${ANT_TEST_OPTS} -Dbuild.test.output.dir=${output_dir} -Dbenchmark.name="${benchmark_pattern}" -Dmaven.test.failure.ignore=true
+
+    # Post-process jmh-result.json to add jdk and arch parameters
+    local jmh_result="${output_dir}/jmh-result.json"
+    if [ -f "${jmh_result}" ]; then
+        python3 -c "
+import json,sys
+with open('${jmh_result}','r') as f:
+    data=json.load(f)
+for r in (data if isinstance(data,list) else [data]):
+    if 'params' not in r:
+        r['params']={}
+    r['params']['jdk']='${java_version}'
+    r['params']['arch']='${arch}'
+with open('${jmh_result}','w') as f:
+    json.dump(data,f)
+"
+    fi
+}
+
 _main() {
   # parameters
   local -r target="${test_target/-repeat/}"
-  local -r split_chunk="${chunk:-'1/1'}" # Chunks formatted as "K/N" for the Kth chunk of N chunks
+  local -r split_chunk="${chunk:-1/1}" # Chunks formatted as "K/N" for the Kth chunk of N chunks
 
   # check split_chunk is compatible with target (if not a regexp)
   if [[ "${_split_chunk}" =~ ^\d+/\d+$ ]] && [[ "1/1" != "${split_chunk}" ]] ; then
     case ${target} in
-      "stress-test" | "fqltool-test" | "microbench" | "cqlsh-test" | "simulator-dtest")
-          error 1 "Target ${target} does not suport splits."
+      "stress-test" | "fqltool-test" | "cqlsh-test" | "simulator-dtest")
+          error 1 "Target ${target} does not support splits."
           ;;
         *)
           ;;
@@ -349,8 +418,7 @@ _main() {
       ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}"
       ;;
     "microbench" | "microbench-test")
-      [[ "x${test_name_regexp}" != "x" ]] && test_name_regexp="-Dbenchmark.name=${test_name_regexp}"
-      ant $target ${ANT_TEST_OPTS} ${test_name_regexp} -Dmaven.test.failure.ignore=true
+      _run_microbench "$target" "${test_name_regexp}" "${split_chunk}"
       ;;
     "test")
       _run_testlist "unit" "testclasslist" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}"

.jenkins/Jenkinsfile

@@ -213,9 +213,9 @@ def tasks() {
     'dtest-upgrade-novnode': [splits: 128, size: 'large'],
     'dtest-upgrade-large': [splits: 32, size: 'large'],
     'dtest-upgrade-large-novnode': [splits: 32, size: 'large'],
-    'microbench-test': [splits: 1, size: 'large'],
+    'microbench-test': [splits: 4, size: 'large', timeout_hours: 2],
     // performance tests need 'cassandra-*large-dedicated' nodes
-    'microbench': [splits: 1, size: 'large', timeout_hours: 6, benchmark: true],
+    'microbench': [splits: 4, size: 'large', timeout_hours: 6, benchmark: true],
   ]
   testSteps.each() {
     it.value.put('type', 'test')
@@ -470,8 +470,8 @@ def test(command, cell) {
                   find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
                   echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
                 """
-              archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
-              copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
+              archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true
+              copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
             }
           } finally {
             cleanAgent(cell.step)
@@ -619,9 +619,10 @@ def generateTestReports() {
           unzip -x -d build/test -q output.zip ) ${teeSuffix}
          """
     } else {
-      copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
+      copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
     }
-    if (fileExists('build/test/output')) {
+    // merge and summarise test reports
+    if (fileExists('build/test/output') && sh(script: 'test -n "$(find build/test/output -type f -name "*.xml.xz" -print -quit)"', returnStatus: true) == 0) {
       // merge splits for each target's test report, other axes are kept separate
       //   TODO parallelised for loop
       //   TODO results_details.tar.xz needs to include all logs for failed tests
@@ -647,11 +648,31 @@ def generateTestReports() {
 
       dir('build/') {
         archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true
-        copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile},test/jmh-result.json')
+        copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile}')
       }
     }
-    if (fileExists('build/test/jmh-result.json')) {
-      jmhReport('build/test/jmh-result.json')
+    processJmhReports()
+  }
+}
+
+def processJmhReports() {
+  if (fileExists('build/test/output') && sh(script: 'test -n "$(find build/test/output -type f -name jmh-result.json -print -quit)"', returnStatus: true) == 0) {
+    sh ''' python3 -c "
+import glob,json
+m=[]
+for f in glob.glob('build/test/output/**/jmh-result.json',recursive=True):
+  d=json.load(open(f))
+  m.extend(d if isinstance(d,list) else [d])
+o=open('build/test/output/combined-result.json','w')
+json.dump(m,o)
+o.close()
+print(f'combined {len(m)} results')
+"
+  '''
+    jmhReport('build/test/output/combined-result.json')
+    dir('build/') {
+      archiveArtifacts artifacts: "test/output/combined-result.json", fingerprint: true
+      copyToNightlies('test/output/combined-result.json')
     }
   }
 }

test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java

@@ -54,6 +54,8 @@ import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 import static org.assertj.core.api.Assertions.assertThat;
 
+// FIXME – rewrite to jmh bench classes
+// https://issues.apache.org/jira/browse/CASSANDRA-17964?focusedCommentId=17680480&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-17680480
 public class CachingBenchTest extends CQLTester
 {
     private static final String STRATEGY = "LeveledCompactionStrategy";

test/microbench/org/apache/cassandra/test/microbench/GcCompactionBenchTest.java

@@ -52,6 +52,8 @@ import org.apache.cassandra.utils.FBUtilities;
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
+// FIXME – rewrite to jmh bench classes
+// https://issues.apache.org/jira/browse/CASSANDRA-17964?focusedCommentId=17680480&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-17680480
 public class GcCompactionBenchTest extends CQLTester
 {
     private static final String SIZE_TIERED_STRATEGY = "SizeTieredCompactionStrategy', 'min_sstable_size' : '0";