Merge branch 'cassandra-4.1' into trunk

.circleci/config-2_1.yml.high_res.patch

@@ -1,6 +1,6 @@
---- config-2_1.yml	2022-05-30 12:09:35.000000000 -0400
-+++ config-2_1.yml.HIGHRES	2022-05-30 12:10:16.000000000 -0400
-@@ -105,14 +105,14 @@
+--- config-2_1.yml	2022-10-15 12:44:53.884924328 +0100
++++ config-2_1.yml.HIGHRES	2022-10-15 12:45:37.970163268 +0100
+@@ -148,14 +148,14 @@
  j8_par_executor: &j8_par_executor
    executor:
      name: java8-executor
@@ -19,7 +19,7 @@
  
  j8_small_executor: &j8_small_executor
    executor:
-@@ -123,62 +123,68 @@
+@@ -166,62 +166,68 @@
  j8_medium_par_executor: &j8_medium_par_executor
    executor:
      name: java8-executor
@@ -102,5 +102,5 @@
 +    exec_resource_class: xlarge
 +  parallelism: 100
  
- j8_with_dtests_jobs: &j8_with_dtests_jobs
+ j8_separate_jobs: &j8_separate_jobs
    jobs:

.circleci/config-2_1.yml.mid_res.patch

@@ -1,6 +1,6 @@
---- config-2_1.yml	2022-05-30 12:09:35.000000000 -0400
-+++ config-2_1.yml.MIDRES	2022-05-30 12:10:10.000000000 -0400
-@@ -105,14 +105,14 @@
+--- config-2_1.yml	2022-10-15 12:44:53.884924328 +0100
++++ config-2_1.yml.MIDRES	2022-10-15 12:45:37.963422950 +0100
+@@ -148,14 +148,14 @@
  j8_par_executor: &j8_par_executor
    executor:
      name: java8-executor
@@ -19,7 +19,7 @@
  
  j8_small_executor: &j8_small_executor
    executor:
-@@ -120,29 +120,41 @@
+@@ -163,29 +163,41 @@
      exec_resource_class: medium
    parallelism: 1
  
@@ -68,7 +68,7 @@
  
  j11_small_executor: &j11_small_executor
    executor:
-@@ -150,35 +162,47 @@
+@@ -193,35 +205,47 @@
      #exec_resource_class: medium
    parallelism: 1
  
@@ -120,120 +120,120 @@
 +    exec_resource_class: large
 +  parallelism: 25
  
- j8_with_dtests_jobs: &j8_with_dtests_jobs
+ j8_separate_jobs: &j8_separate_jobs
    jobs:
-@@ -873,7 +897,7 @@
+@@ -1045,7 +1069,7 @@
            target: testclasslist-system-keyspace-directory
  
-   j8_dtests-with-vnodes:
+   j8_dtests_vnode:
 -    <<: *j8_par_executor
 +    <<: *j8_large_par_executor
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -887,7 +911,7 @@
+@@ -1059,7 +1083,7 @@
            pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
  
-   j11_dtests-with-vnodes:
+   j11_dtests_vnode:
 -    <<: *j11_par_executor
 +    <<: *j11_large_par_executor
      steps:
      - attach_workspace:
          at: /home/cassandra
-@@ -902,7 +926,7 @@
+@@ -1074,7 +1098,7 @@
          pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
  
-   j8_dtests-no-vnodes:
+   j8_dtests:
 -    <<: *j8_par_executor
 +    <<: *j8_large_par_executor
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -916,7 +940,7 @@
+@@ -1088,7 +1112,7 @@
            pytest_extra_args: '--skip-resource-intensive-tests'
  
-   j11_dtests-no-vnodes:
+   j11_dtests:
 -    <<: *j11_par_executor
 +    <<: *j11_large_par_executor
      steps:
      - attach_workspace:
          at: /home/cassandra
-@@ -931,7 +955,7 @@
+@@ -1103,7 +1127,7 @@
          pytest_extra_args: '--skip-resource-intensive-tests'
  
-   j8_upgradetests-no-vnodes:
+   j8_upgrade_dtests:
 -    <<: *j8_par_executor
 +    <<: *j8_very_large_par_executor
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -945,7 +969,7 @@
+@@ -1117,7 +1141,7 @@
            pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
  
-   j8_cqlsh-dtests-py3-with-vnodes:
+   j8_cqlsh_dtests_py3_vnode:
 -    <<: *j8_par_executor
 +    <<: *j8_large_par_executor
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -960,7 +984,7 @@
+@@ -1132,7 +1156,7 @@
            extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
  
-   j8_cqlsh-dtests-py38-with-vnodes:
+   j8_cqlsh_dtests_py38_vnode:
 -    <<: *j8_par_executor
 +    <<: *j8_large_par_executor
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -978,7 +1002,7 @@
+@@ -1150,7 +1174,7 @@
            python_version: '3.8'
  
-   j8_cqlsh-dtests-py3-no-vnodes:
+   j8_cqlsh_dtests_py3:
 -    <<: *j8_par_executor
 +    <<: *j8_large_par_executor
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -993,7 +1017,7 @@
+@@ -1165,7 +1189,7 @@
            extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
  
-   j8_cqlsh-dtests-py38-no-vnodes:
+   j8_cqlsh_dtests_py38:
 -    <<: *j8_par_executor
 +    <<: *j8_large_par_executor
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1011,7 +1035,7 @@
+@@ -1183,7 +1207,7 @@
            python_version: '3.8'
  
-   j11_cqlsh-dtests-py3-with-vnodes:
+   j11_cqlsh_dtests_py3_vnode:
 -    <<: *j11_par_executor
 +    <<: *j11_large_par_executor
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1026,7 +1050,7 @@
+@@ -1198,7 +1222,7 @@
            extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
  
-   j11_cqlsh-dtests-py38-with-vnodes:
+   j11_cqlsh_dtests_py38_vnode:
 -    <<: *j11_par_executor
 +    <<: *j11_large_par_executor
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1044,7 +1068,7 @@
+@@ -1216,7 +1240,7 @@
            python_version: '3.8'
  
-   j11_cqlsh-dtests-py3-no-vnodes:
+   j11_cqlsh_dtests_py3:
 -    <<: *j11_par_executor
 +    <<: *j11_large_par_executor
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1059,7 +1083,7 @@
+@@ -1231,7 +1255,7 @@
            extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
  
-   j11_cqlsh-dtests-py38-no-vnodes:
+   j11_cqlsh_dtests_py38:
 -    <<: *j11_par_executor
 +    <<: *j11_large_par_executor
      steps:

.circleci/generate.sh

@@ -18,6 +18,7 @@
 #
 
 BASEDIR=`dirname $0`
+BASE_BRANCH=trunk
 
 die ()
 {
@@ -38,16 +39,30 @@ print_help()
   echo "   -e <key=value> Environment variables to be used in the generated config.yml, e.g.:"
   echo "                   -e DTEST_BRANCH=CASSANDRA-8272"
   echo "                   -e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git"
-  echo "                   -e REPEATED_UTEST_TARGET=testsome"
-  echo "                   -e REPEATED_UTEST_CLASS=org.apache.cassandra.cql3.ViewTest"
-  echo "                   -e REPEATED_UTEST_METHODS=testCompoundPartitionKey,testStaticTable"
-  echo "                   -e REPEATED_UTEST_VNODES=false"
-  echo "                   -e REPEATED_UTEST_COUNT=100"
-  echo "                   -e REPEATED_UTEST_STOP_ON_FAILURE=false"
-  echo "                   -e REPEATED_DTEST_NAME=cqlsh_tests/test_cqlsh.py::TestCqlshSmoke"
-  echo "                   -e REPEATED_DTEST_VNODES=false"
-  echo "                   -e REPEATED_DTEST_COUNT=100"
-  echo "                   -e REPEATED_DTEST_STOP_ON_FAILURE=false"
+  echo "                   -e REPEATED_TESTS_STOP_ON_FAILURE=false"
+  echo "                   -e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest#testCountersTable"
+  echo "                   -e REPEATED_UTESTS_COUNT=500"
+  echo "                   -e REPEATED_UTESTS_FQLTOOL=org.apache.cassandra.fqltool.FQLCompareTest"
+  echo "                   -e REPEATED_UTESTS_FQLTOOL_COUNT=500"
+  echo "                   -e REPEATED_UTESTS_LONG=org.apache.cassandra.db.commitlog.CommitLogStressTest"
+  echo "                   -e REPEATED_UTESTS_LONG_COUNT=100"
+  echo "                   -e REPEATED_UTESTS_STRESS=org.apache.cassandra.stress.generate.DistributionGaussianTest"
+  echo "                   -e REPEATED_UTESTS_STRESS_COUNT=500"
+  echo "                   -e REPEATED_SIMULATOR_DTESTS=org.apache.cassandra.simulator.test.TrivialSimulationTest"
+  echo "                   -e REPEATED_SIMULATOR_DTESTS_COUNT=500"
+  echo "                   -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest"
+  echo "                   -e REPEATED_JVM_DTESTS_COUNT=500"
+  echo "                   -e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest"
+  echo "                   -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500"
+  echo "                   -e REPEATED_DTESTS=cdc_test.py cqlsh_tests/test_cqlsh.py::TestCqlshSmoke"
+  echo "                   -e REPEATED_DTESTS_COUNT=500"
+  echo "                   -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py upgrade_tests/paging_test.py"
+  echo "                   -e REPEATED_UPGRADE_DTESTS_COUNT=25"
+  echo "                   -e REPEATED_ANT_TEST_TARGET=testsome"
+  echo "                   -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest"
+  echo "                   -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable"
+  echo "                   -e REPEATED_ANT_TEST_VNODES=false"
+  echo "                   -e REPEATED_ANT_TEST_COUNT=500"
   echo "                  For the complete list of environment variables, please check the"
   echo "                  list of examples in config-2_1.yml and/or the documentation."
   echo "                  If you want to specify multiple environment variables simply add"
@@ -96,23 +111,30 @@ if $has_env_vars && $check_env_vars; then
     key=$(echo $entry | tr "=" "\n" | head -n 1)
     if [ "$key" != "DTEST_REPO" ] &&
        [ "$key" != "DTEST_BRANCH" ] &&
-       [ "$key" != "REPEATED_UTEST_TARGET" ] &&
-       [ "$key" != "REPEATED_UTEST_CLASS" ] &&
-       [ "$key" != "REPEATED_UTEST_METHODS" ] &&
-       [ "$key" != "REPEATED_UTEST_VNODES" ] &&
-       [ "$key" != "REPEATED_UTEST_COUNT" ] &&
-       [ "$key" != "REPEATED_UTEST_STOP_ON_FAILURE" ] &&
-       [ "$key" != "REPEATED_DTEST_NAME" ] &&
-       [ "$key" != "REPEATED_DTEST_VNODES" ] &&
-       [ "$key" != "REPEATED_DTEST_COUNT" ] &&
-       [ "$key" != "REPEATED_DTEST_STOP_ON_FAILURE" ] &&
-       [ "$key" != "REPEATED_UPGRADE_DTEST_NAME" ] &&
-       [ "$key" != "REPEATED_UPGRADE_DTEST_COUNT" ] &&
-       [ "$key" != "REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE" ] &&
-       [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_CLASS" ] &&
-       [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_METHODS" ] &&
-       [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_COUNT" ] &&
-       [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE" ]; then
+       [ "$key" != "REPEATED_TESTS_STOP_ON_FAILURE" ] &&
+       [ "$key" != "REPEATED_UTESTS" ] &&
+       [ "$key" != "REPEATED_UTESTS_COUNT" ] &&
+       [ "$key" != "REPEATED_UTESTS_FQLTOOL" ] &&
+       [ "$key" != "REPEATED_UTESTS_FQLTOOL_COUNT" ] &&
+       [ "$key" != "REPEATED_UTESTS_LONG" ] &&
+       [ "$key" != "REPEATED_UTESTS_LONG_COUNT" ] &&
+       [ "$key" != "REPEATED_UTESTS_STRESS" ] &&
+       [ "$key" != "REPEATED_UTESTS_STRESS_COUNT" ] &&
+       [ "$key" != "REPEATED_SIMULATOR_DTESTS" ] &&
+       [ "$key" != "REPEATED_SIMULATOR_DTESTS_COUNT" ] &&
+       [ "$key" != "REPEATED_JVM_DTESTS" ] &&
+       [ "$key" != "REPEATED_JVM_DTESTS_COUNT" ] &&
+       [ "$key" != "REPEATED_JVM_UPGRADE_DTESTS" ]  &&
+       [ "$key" != "REPEATED_JVM_UPGRADE_DTESTS_COUNT" ]  &&
+       [ "$key" != "REPEATED_DTESTS" ] &&
+       [ "$key" != "REPEATED_DTESTS_COUNT" ] &&
+       [ "$key" != "REPEATED_UPGRADE_DTESTS" ] &&
+       [ "$key" != "REPEATED_UPGRADE_DTESTS_COUNT" ] &&
+       [ "$key" != "REPEATED_ANT_TEST_TARGET" ] &&
+       [ "$key" != "REPEATED_ANT_TEST_CLASS" ] &&
+       [ "$key" != "REPEATED_ANT_TEST_METHODS" ] &&
+       [ "$key" != "REPEATED_ANT_TEST_VNODES" ] &&
+       [ "$key" != "REPEATED_ANT_TEST_COUNT" ]; then
       die "Unrecognised environment variable name: $key"
     fi
   done
@@ -169,16 +191,108 @@ elif (!($has_env_vars)); then
   print_help
 fi
 
+# add new or modified tests to the sets of tests to be repeated
+if (!($all)); then
+  add_diff_tests ()
+  {
+    dir="${BASEDIR}/../${2}"
+    diff=$(git --no-pager diff --name-only --diff-filter=AMR ${BASE_BRANCH}...HEAD ${dir})
+    tests=$( echo "$diff" \
+           | grep "Test\\.java" \
+           | sed -e "s/\\.java//" \
+           | sed -e "s,^${2},," \
+           | tr  '/' '.' \
+           | grep ${3} )
+    for test in $tests; do
+      echo "  $test"
+      has_env_vars=true
+      if grep -q "${1}=" <<< "$env_vars"; then
+        env_vars=$(sed -e "s/${1}=/${1}=${test},/" <<< $env_vars)
+      elif [[ $env_vars == "" ]]; then
+        env_vars="${1}=${test}"
+      else
+        env_vars="$env_vars|${1}=${test}"
+      fi
+    done
+  }
+
+  echo
+  echo "Detecting new or modified tests with git diff --diff-filter=AMR ${BASE_BRANCH}...HEAD:"
+  add_diff_tests "REPEATED_UTESTS" "test/unit/" "org.apache.cassandra"
+  add_diff_tests "REPEATED_UTESTS_LONG" "test/long/" "org.apache.cassandra"
+  add_diff_tests "REPEATED_UTESTS_STRESS" "tools/stress/test/unit/" "org.apache.cassandra.stress"
+  add_diff_tests "REPEATED_UTESTS_FQLTOOL" "tools/fqltool/test/unit/" "org.apache.cassandra.fqltool"
+  add_diff_tests "REPEATED_SIMULATOR_DTESTS" "test/simulator/test/" "org.apache.cassandra.simulator.test"
+  add_diff_tests "REPEATED_JVM_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.test"
+  add_diff_tests "REPEATED_JVM_UPGRADE_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.upgrade"
+fi
+
 # replace environment variables
 if $has_env_vars; then
+  echo
+  echo "Setting environment variables:"
   IFS='='
   echo "$env_vars" | tr '|' '\n' | while read entry; do
     set -- $entry
     key=$1
     val=$2
-    echo "Setting environment variable $key: $val"
+    echo "  $key: $val"
     sed -i.bak "s|- $key:.*|- $key: $val|" $BASEDIR/config.yml
   done
   unset IFS
 fi
 
+# define function to remove unneeded jobs
+delete_job()
+{
+  delete_yaml_block()
+  {
+    sed -Ei.bak "/^    - ${1}/,/^    [^[:space:]]+|^  [^[:space:]]+/{//!d;}" $BASEDIR/config.yml
+    sed -Ei.bak "/^    - ${1}/d" $BASEDIR/config.yml
+  }
+  delete_yaml_block "${1}"
+  delete_yaml_block "start_${1}"
+}
+
+# removed unneeded repeated jobs
+if [[ $env_vars != *"REPEATED_UTESTS="* ]]; then
+  delete_job "j8_unit_tests_repeat"
+  delete_job "j11_unit_tests_repeat"
+  delete_job "utests_compression_repeat"
+  delete_job "utests_system_keyspace_directory_repeat"
+fi
+if [[ $env_vars != *"REPEATED_UTESTS_LONG="* ]]; then
+  delete_job "utests_long_repeat"
+fi
+if [[ $env_vars != *"REPEATED_UTESTS_STRESS="* ]]; then
+  delete_job "utests_stress_repeat"
+fi
+if [[ $env_vars != *"REPEATED_UTESTS_FQLTOOL="* ]]; then
+  delete_job "utests_fqltool_repeat"
+fi
+if [[ $env_vars != *"REPEATED_SIMULATOR_DTESTS="* ]]; then
+  delete_job "j8_simulator_dtests_repeat"
+fi
+if [[ $env_vars != *"REPEATED_JVM_DTESTS="* ]]; then
+  delete_job "j8_jvm_dtests_repeat"
+  delete_job "j8_jvm_dtests_vnode_repeat"
+  delete_job "j11_jvm_dtests_repeat"
+  delete_job "j11_jvm_dtests_vnode_repeat"
+fi
+if [[ $env_vars != *"REPEATED_JVM_UPGRADE_DTESTS="* ]]; then
+  delete_job "start_jvm_upgrade_dtests_repeat"
+  delete_job "j8_jvm_upgrade_dtests_repeat"
+fi
+if [[ $env_vars != *"REPEATED_DTESTS="* ]]; then
+  delete_job "j8_dtests_repeat"
+  delete_job "j8_dtests_vnode_repeat"
+  delete_job "j11_dtests_repeat"
+  delete_job "j11_dtests_vnode_repeat"
+fi
+if [[ $env_vars != *"REPEATED_UPGRADE_DTESTS="* ]]; then
+  delete_job "j8_upgrade_dtests_repeat"
+fi
+if [[ $env_vars != *"REPEATED_ANT_TEST_CLASS="* ]]; then
+  delete_job "j8_repeated_ant_test"
+  delete_job "j11_repeated_ant_test"
+fi

.circleci/readme.md

@@ -22,7 +22,7 @@
 
 This directory contains the configuration for CircleCI continous integration platform.
 The file `config.yml` is the configuration file that is read by CircleCI. This file is
-automatically generated by the `generate.sh` script from the `config-2_1.yml` file. 
+automatically generated by the `generate.sh` script from the `config-2_1.yml` file.
 
 The provided `config.yml` file uses low resources so users of the CircleCI free tier can
 use it. Additionally, there are three versions of this file using different resources so
@@ -43,33 +43,99 @@ requires the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to
 installed.
 
 ## Setting environment variables
-Both `config-2_1.yml` and `config.yml` files contain a set of environment variables 
-defining things like what dtest repo and branch to use, what tests could be repeatedly 
+Both `config-2_1.yml` and `config.yml` files contain a set of environment variables
+defining things like what dtest repo and branch to use, what tests could be repeatedly
 run, etc.
 
-These environment variables can be directly edited in the `config.yml` file, although if 
-you do this you should take into account that the entire set of env vars is repeated on 
+These environment variables can be directly edited in the `config.yml` file, although if
+you do this you should take into account that the entire set of env vars is repeated on
 every job. 
 
-A probably better approach is editing them in `config-2_1.yml` and then regenerate the 
+A probably better approach is editing them in `config-2_1.yml` and then regenerate the
 `config.yml` file using the `generate.sh` script. You can also directly pass environment
-variable values to the `generate.sh` script with the `-e` flag. For example, to set the 
+variable values to the `generate.sh` script with the `-e` flag. For example, to set the
 dtest repo and branch with MIDRES config you can run:
 
 ```
 generate.sh -m \
   -e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git \
-  -e DTEST_BRANCH=CASSANDRA-8272 
-```
-
-Or you can set the test multiplexer for repeating a specific test with HIGHRES:
+  -e DTEST_BRANCH=CASSANDRA-8272
 
 ```
-generate.sh -h \
-  -e REPEATED_UTEST_TARGET=testsome \
-  -e REPEATED_UTEST_CLASS=org.apache.cassandra.cql3.ViewTest \
-  -e REPEATED_UTEST_METHODS=testCompoundPartitionKey,testStaticTable \
-  -e REPEATED_UTEST_COUNT=100
+
+## Running tests in a loop
+Running the `generate.sh` script with use `git diff` to find the new or modified tests.
+The script will then create jobs to run each of these new or modified tests for a certain
+number of times, to verify that they are stable. You can use environment variables to
+specify the number of iterations of each type of test:
+```
+generate.sh -m \
+  -e REPEATED_UTESTS_COUNT=500 \
+  -e REPEATED_UTESTS_FQLTOOL_COUNT=500 \
+  -e REPEATED_UTESTS_LONG_COUNT=100 \
+  -e REPEATED_UTESTS_STRESS_COUNT=500 \
+  -e REPEATED_SIMULATOR_DTESTS_COUNT=500 \
+  -e REPEATED_JVM_DTESTS_COUNT=500 \
+  -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500 \
+  -e REPEATED_DTESTS_COUNT=500 \
+  -e REPEATED_UPGRADE_DTESTS_COUNT=25 \
+  -e REPEATED_ANT_TEST_COUNT=500
+```
+You can also specify whether the iteration should fail on the first test failure:
+```
+generate.sh -m -e REPEATED_TESTS_STOP_ON_FAILURE=false
+```
+In addition to the automatically detected tests, it's also possible to provide lists of
+specific tests to be repeated:
+```
+generate.sh -m \
+  -e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest \
+  -e REPEATED_UTESTS_FQLTOOL=org.apache.cassandra.fqltool.FQLCompareTest \
+  -e REPEATED_UTESTS_LONG=org.apache.cassandra.io.sstable.CQLSSTableWriterLongTest#testWideRow \
+  -e REPEATED_UTESTS_STRESS=org.apache.cassandra.stress.generate.DistributionGaussianTest \
+  -e REPEATED_SIMULATOR_DTESTS=org.apache.cassandra.simulator.test.TrivialSimulationTest \
+  -e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \
+  -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest#testPaging \
+  -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py \
+  -e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest
+```
+For particular Ant test targets that are not included in the regular test suites, you can
+use the `run_repeated_utest` job:
+```
+generate.sh -m \
+  -e REPEATED_ANT_TEST_TARGET=test-cdc \
+  -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest \
+  -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable \
+  -e REPEATED_ANT_TEST_VNODES=false \
+  -e REPEATED_ANT_TEST_COUNT=500
+```
+Putting all together, you can have runs as complex as:
+```
+generate.sh -m \
+  -e REPEATED_TESTS_STOP_ON_FAILURE=true \
+  -e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest \
+  -e REPEATED_UTESTS_COUNT=500 \
+  -e REPEATED_UTESTS_FQLTOOL=org.apache.cassandra.fqltool.FQLCompareTest \
+  -e REPEATED_UTESTS_FQLTOOL_COUNT=500 \
+  -e REPEATED_UTESTS_LONG=org.apache.cassandra.io.sstable.CQLSSTableWriterLongTest#testWideRow \
+  -e REPEATED_UTESTS_LONG_COUNT=100 \
+  -e REPEATED_UTESTS_STRESS=org.apache.cassandra.stress.generate.DistributionGaussianTest \
+  -e REPEATED_UTESTS_STRESS_COUNT=500 \
+  -e REPEATED_SIMULATOR_DTESTS=org.apache.cassandra.simulator.test.TrivialSimulationTest \
+  -e REPEATED_SIMULATOR_DTESTS_COUNT=500 \
+  -e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \
+  -e REPEATED_DTESTS_COUNT=500 \
+  -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest#testPaging \
+  -e REPEATED_JVM_DTESTS_COUNT=500 \
+  -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py \
+  -e REPEATED_UPGRADE_DTESTS_COUNT=25 \
+  -e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest \
+  -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500 \
+  -e REPEATED_ANT_TEST_TARGET=test-cdc \
+  -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest \
+  -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable \
+  -e REPEATED_ANT_TEST_VNODES=false \
+  -e REPEATED_ANT_TEST_COUNT=500
 ```
 
 ## Updating the config
@@ -78,7 +144,7 @@ the `config.yml` file manually. Instead, you should edit the `config-2_1.yml` fi
 regenerate the `config.yml`, `config.yml.LOWRES`, `config.yml.MIDRES` and `config.yml.HIGHRES`
 files by runnining the `generate.sh` script without any flags. For using this script you
 need to install the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install).
- 
+
 As for temporal changes done while working in a patch, such as pointing to you dtest repo or
 running a test repeatedly, you can either directly edit `config.yml` or edit `config-2_1.yml`
 and then regenerate `config.yml` with the `generate.sh` script using a `-l`/`-m`/`-h` flag.
@@ -116,4 +182,3 @@ Read below for details how to generate the files manually without the `generate.
    cat license.yml config.yml.MIDRES > config.yml.MIDRES.new && mv config.yml.MIDRES.new config.yml.MIDRES
    ```
 9. finally, remember to update the config.yml
-