Merge branch 'cassandra-4.1' into trunk

.circleci/config-2_1.yml

@@ -114,6 +114,15 @@ default_env_vars: &default_env_vars
     # The number of times that the manually specified Python dtests should be run.
     REPEATED_DTESTS_COUNT: 500
 
+    # Comma-separated list of Python large dtests that should be repeatedly run, for example:
+    # REPEATED_LARGE_DTESTS: consistency_test.py
+    # REPEATED_LARGE_DTESTS: consistency_test.py::TestAvailability
+    # REPEATED_LARGE_DTESTS: consistency_test.py::TestAvailability::test_network_topology_strategy
+    # REPEATED_LARGE_DTESTS: consistency_test.py,materialized_views_test.py
+    REPEATED_LARGE_DTESTS:
+    # The number of times that the manually specified Python large dtests should be run.
+    REPEATED_LARGE_DTESTS_COUNT: 100
+
     # Comma-separated list of Python dtests that should be repeatedly run, for example:
     # REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py
     # REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py::TestCQLNodes2RF1_Upgrade_current_4_0_x_To_indev_4_0_x
@@ -194,6 +203,12 @@ j11_small_executor: &j11_small_executor
     #exec_resource_class: medium
   parallelism: 1
 
+j11_medium_par_executor: &j11_medium_par_executor
+  executor:
+    name: java11-executor
+    #exec_resource_class: xlarge
+  parallelism: 1
+
 j11_seq_executor: &j11_seq_executor
   executor:
     name: java11-executor
@@ -274,6 +289,30 @@ j8_separate_jobs: &j8_separate_jobs
         requires:
           - start_j8_jvm_dtests_vnode_repeat
           - j8_build
+    - start_j11_jvm_dtests:
+        type: approval
+    - j11_jvm_dtests:
+        requires:
+          - start_j11_jvm_dtests
+          - j8_build
+    - start_j11_jvm_dtests_vnode:
+        type: approval
+    - j11_jvm_dtests_vnode:
+        requires:
+          - start_j11_jvm_dtests_vnode
+          - j8_build
+    - start_j11_jvm_dtests_repeat:
+        type: approval
+    - j11_jvm_dtests_repeat:
+        requires:
+          - start_j11_jvm_dtests_repeat
+          - j8_build
+    - start_j11_jvm_dtests_vnode_repeat:
+        type: approval
+    - j11_jvm_dtests_vnode_repeat:
+        requires:
+          - start_j11_jvm_dtests_vnode_repeat
+          - j8_build
     - start_j8_simulator_dtests:
         type: approval
     - j8_simulator_dtests:
@@ -292,6 +331,24 @@ j8_separate_jobs: &j8_separate_jobs
         requires:
           - start_j8_cqlshlib_tests
           - j8_build
+    - start_j8_cqlshlib_cython_tests:
+        type: approval
+    - j8_cqlshlib_cython_tests:
+        requires:
+          - start_j8_cqlshlib_cython_tests
+          - j8_build
+    - start_j11_cqlshlib_tests:
+        type: approval
+    - j11_cqlshlib_tests:
+        requires:
+          - start_j11_cqlshlib_tests
+          - j8_build
+    - start_j11_cqlshlib_cython_tests:
+        type: approval
+    - j11_cqlshlib_cython_tests:
+        requires:
+          - start_j11_cqlshlib_cython_tests
+          - j8_build
     # Java 11 unit tests
     - start_j11_unit_tests:
         type: approval
@@ -492,7 +549,7 @@ j8_separate_jobs: &j8_separate_jobs
         requires:
           - start_jvm_upgrade_dtests_repeat
           - j8_dtest_jars_build
-    # Java 8 dtests
+    # Python DTests
     - start_j8_dtests:
         type: approval
     - j8_dtests:
@@ -544,6 +601,55 @@ j8_separate_jobs: &j8_separate_jobs
         requires:
           - start_j11_dtests_offheap_repeat
           - j8_build
+    # Python large DTests
+    - start_j8_dtests_large:
+        type: approval
+    - j8_dtests_large:
+        requires:
+          - start_j8_dtests_large
+          - j8_build
+    - start_j8_dtests_large_repeat:
+        type: approval
+    - j8_dtests_large_repeat:
+        requires:
+          - start_j8_dtests_large_repeat
+          - j8_build
+    - start_j8_dtests_large_vnode:
+        type: approval
+    - j8_dtests_large_vnode:
+        requires:
+          - start_j8_dtests_large_vnode
+          - j8_build
+    - start_j8_dtests_large_vnode_repeat:
+        type: approval
+    - j8_dtests_large_vnode_repeat:
+        requires:
+          - start_j8_dtests_large_vnode_repeat
+          - j8_build
+    - start_j11_dtests_large:
+        type: approval
+    - j11_dtests_large:
+        requires:
+          - start_j11_dtests_large
+          - j8_build
+    - start_j11_dtests_large_repeat:
+        type: approval
+    - j11_dtests_large_repeat:
+        requires:
+          - start_j11_dtests_large_repeat
+          - j8_build
+    - start_j11_dtests_large_vnode:
+        type: approval
+    - j11_dtests_large_vnode:
+        requires:
+          - start_j11_dtests_large_vnode
+          - j8_build
+    - start_j11_dtests_large_vnode_repeat:
+        type: approval
+    - j11_dtests_large_vnode_repeat:
+        requires:
+          - start_j11_dtests_large_vnode_repeat
+          - j8_build
     # Java 8 upgrade tests
     - start_upgrade_dtests:
         type: approval
@@ -691,9 +797,30 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs
     - j8_jvm_dtests_vnode_repeat:
         requires:
           - j8_build
+    - j11_jvm_dtests:
+        requires:
+          - j8_build
+    - j11_jvm_dtests_repeat:
+        requires:
+          - j8_build
+    - j11_jvm_dtests_vnode:
+        requires:
+          - j8_build
+    - j11_jvm_dtests_vnode_repeat:
+        requires:
+          - j8_build
     - j8_cqlshlib_tests:
         requires:
           - j8_build
+    - j8_cqlshlib_cython_tests:
+        requires:
+          - j8_build
+    - j11_cqlshlib_tests:
+        requires:
+          - j8_build
+    - j11_cqlshlib_cython_tests:
+        requires:
+          - j8_build
     # Java 11 unit tests
     - j11_unit_tests:
         requires:
@@ -838,7 +965,7 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs
     - j8_jvm_upgrade_dtests_repeat:
         requires:
           - j8_dtest_jars_build
-    # Java 8 dtests
+    # Python DTests
     - j8_dtests:
         requires:
           - j8_build
@@ -884,6 +1011,55 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs
         requires:
           - start_j11_dtests_offheap
           - j8_build
+    # Large Python DTests
+    - start_j8_dtests_large:
+        type: approval
+    - j8_dtests_large:
+        requires:
+          - start_j8_dtests_large
+          - j8_build
+    - start_j8_dtests_large_repeat:
+        type: approval
+    - j8_dtests_large_repeat:
+        requires:
+          - start_j8_dtests_large_repeat
+          - j8_build
+    - start_j8_dtests_large_vnode:
+        type: approval
+    - j8_dtests_large_vnode:
+        requires:
+          - start_j8_dtests_large_vnode
+          - j8_build
+    - start_j8_dtests_large_vnode_repeat:
+        type: approval
+    - j8_dtests_large_vnode_repeat:
+        requires:
+          - start_j8_dtests_large_vnode_repeat
+          - j8_build
+    - start_j11_dtests_large:
+        type: approval
+    - j11_dtests_large:
+        requires:
+          - start_j11_dtests_large
+          - j8_build
+    - start_j11_dtests_large_repeat:
+        type: approval
+    - j11_dtests_large_repeat:
+        requires:
+          - start_j11_dtests_large_repeat
+          - j8_build
+    - start_j11_dtests_large_vnode:
+        type: approval
+    - j11_dtests_large_vnode:
+        requires:
+          - start_j11_dtests_large_vnode
+          - j8_build
+    - start_j11_dtests_large_vnode_repeat:
+        type: approval
+    - j11_dtests_large_vnode_repeat:
+        requires:
+          - start_j11_dtests_large_vnode_repeat
+          - j8_build
     # Java 8 upgrade tests (on request)
     - start_upgrade_dtests:
         type: approval
@@ -994,6 +1170,12 @@ j11_separate_jobs: &j11_separate_jobs
         requires:
           - start_j11_cqlshlib_tests
           - j11_build
+    - start_j11_cqlshlib_cython_tests:
+        type: approval
+    - j11_cqlshlib_cython_tests:
+        requires:
+          - start_j11_cqlshlib_cython_tests
+          - j11_build
     # Java 11 dtests
     - start_j11_dtests:
         type: approval
@@ -1019,6 +1201,30 @@ j11_separate_jobs: &j11_separate_jobs
         requires:
           - start_j11_dtests_offheap_repeat
           - j11_build
+    - start_j11_dtests_large:
+        type: approval
+    - j11_dtests_large:
+        requires:
+          - start_j11_dtests_large
+          - j11_build
+    - start_j11_dtests_large_repeat:
+        type: approval
+    - j11_dtests_large_repeat:
+        requires:
+          - start_j11_dtests_large_repeat
+          - j11_build
+    - start_j11_dtests_large_vnode:
+        type: approval
+    - j11_dtests_large_vnode:
+        requires:
+          - start_j11_dtests_large_vnode
+          - j11_build
+    - start_j11_dtests_large_vnode_repeat:
+        type: approval
+    - j11_dtests_large_vnode_repeat:
+        requires:
+          - start_j11_dtests_large_vnode_repeat
+          - j11_build
     - start_j11_cqlsh_tests:
         type: approval
     - j11_cqlsh_dtests_py3:
@@ -1181,7 +1387,7 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs
     - j11_cqlshlib_tests:
         requires:
           - j11_build
-    - j11_cqlshlib_tests:
+    - j11_cqlshlib_cython_tests:
         requires:
           - j11_build
     - j11_dtests:
@@ -1208,6 +1414,30 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs
         requires:
           - start_j11_dtests_offheap_repeat
           - j11_build
+    - start_j11_dtests_large:
+        type: approval
+    - j11_dtests_large:
+        requires:
+          - start_j11_dtests_large
+          - j11_build
+    - start_j11_dtests_large_repeat:
+        type: approval
+    - j11_dtests_large_repeat:
+        requires:
+          - start_j11_dtests_large_repeat
+          - j11_build
+    - start_j11_dtests_large_vnode:
+        type: approval
+    - j11_dtests_large_vnode:
+        requires:
+          - start_j11_dtests_large_vnode
+          - j11_build
+    - start_j11_dtests_large_vnode_repeat:
+        type: approval
+    - j11_dtests_large_vnode_repeat:
+        requires:
+          - start_j11_dtests_large_vnode_repeat
+          - j11_build
     - j11_cqlsh_dtests_py3:
         requires:
           - j11_build
@@ -1476,6 +1706,13 @@ jobs:
           at: /home/cassandra
       - run_cqlshlib_tests
 
+  j8_cqlshlib_cython_tests:
+    <<: *j8_small_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - run_cqlshlib_cython_tests
+
   j11_cqlshlib_tests:
     <<: *j11_small_executor
     steps:
@@ -1483,6 +1720,13 @@ jobs:
           at: /home/cassandra
       - run_cqlshlib_tests
 
+  j11_cqlshlib_cython_tests:
+    <<: *j11_small_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - run_cqlshlib_cython_tests
+
   j8_utests_long:
     <<: *j8_seq_executor
     steps:
@@ -1910,6 +2154,62 @@ jobs:
           extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8'
           python_version: '3.8'
 
+  j8_dtests_large_vnode:
+    <<: *j8_par_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - clone_dtest
+      - create_venv
+      - create_dtest_containers:
+          file_tag: j8_large_with_vnodes
+          run_dtests_extra_args: '--use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests'
+      - run_dtests:
+          file_tag: j8_large_with_vnodes
+          pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+
+  j8_dtests_large:
+    <<: *j8_par_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - clone_dtest
+      - create_venv
+      - create_dtest_containers:
+          file_tag: j8_large_without_vnodes
+          run_dtests_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+      - run_dtests:
+          file_tag: j8_large_without_vnodes
+          pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+
+  j11_dtests_large_vnode:
+    <<: *j11_par_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - clone_dtest
+      - create_venv
+      - create_dtest_containers:
+          file_tag: j11_large_with_vnodes
+          run_dtests_extra_args: '--use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests'
+      - run_dtests:
+          file_tag: j11_large_with_vnodes
+          pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+
+  j11_dtests_large:
+    <<: *j11_par_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - clone_dtest
+      - create_venv
+      - create_dtest_containers:
+          file_tag: j11_large_without_vnodes
+          run_dtests_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+      - run_dtests:
+          file_tag: j11_large_without_vnodes
+          pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+
   j8_unit_tests_repeat:
     <<: *j8_repeated_utest_executor
     steps:
@@ -2157,6 +2457,36 @@ jobs:
           stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
           extra_dtest_args: "--use-off-heap-memtables --skip-resource-intensive-tests"
 
+  j8_dtests_large_repeat:
+    <<: *j8_repeated_dtest_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - clone_dtest
+      - create_venv
+      - run_repeated_dtest:
+          tests: ${REPEATED_LARGE_DTESTS}
+          vnodes: "false"
+          upgrade: "false"
+          count: ${REPEATED_LARGE_DTESTS_COUNT}
+          stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+          extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
+
+  j8_dtests_large_vnode_repeat:
+    <<: *j8_repeated_dtest_executor
+    steps:
+    - attach_workspace:
+        at: /home/cassandra
+    - clone_dtest
+    - create_venv
+    - run_repeated_dtest:
+        tests: ${REPEATED_LARGE_DTESTS}
+        vnodes: "true"
+        upgrade: "false"
+        count: ${REPEATED_LARGE_DTESTS_COUNT}
+        stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+        extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
+
   j11_dtests_repeat:
     <<: *j11_repeated_dtest_executor
     steps:
@@ -2202,6 +2532,36 @@ jobs:
           stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
           extra_dtest_args: "--use-off-heap-memtables --skip-resource-intensive-tests"
 
+  j11_dtests_large_repeat:
+    <<: *j11_repeated_dtest_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - clone_dtest
+      - create_venv
+      - run_repeated_dtest:
+          tests: ${REPEATED_LARGE_DTESTS}
+          vnodes: "false"
+          upgrade: "false"
+          count: ${REPEATED_LARGE_DTESTS_COUNT}
+          stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+          extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
+
+  j11_dtests_large_vnode_repeat:
+    <<: *j11_repeated_dtest_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - clone_dtest
+      - create_venv
+      - run_repeated_dtest:
+          tests: ${REPEATED_LARGE_DTESTS}
+          vnodes: "true"
+          upgrade: "false"
+          count: ${REPEATED_LARGE_DTESTS_COUNT}
+          stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+          extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
+
   j8_upgrade_dtests_repeat:
     <<: *j8_repeated_upgrade_dtest_executor
     steps:
@@ -2433,12 +2793,30 @@ commands:
         command: |
           export PATH=$JAVA_HOME/bin:$PATH
           time mv ~/cassandra /tmp
-          cd /tmp/cassandra/pylib
-          ./cassandra-cqlsh-tests.sh ..
+          cd /tmp/cassandra/
+          ./pylib/cassandra-cqlsh-tests.sh $(pwd)
         no_output_timeout: <<parameters.no_output_timeout>>
     - store_test_results:
         path: /tmp/cassandra/pylib
 
+  run_cqlshlib_cython_tests:
+    parameters:
+      no_output_timeout:
+        type: string
+        default: 15m
+    steps:
+      - run:
+          name: Run cqlshlib Unit Tests
+          command: |
+            export PATH=$JAVA_HOME/bin:$PATH
+            export cython="yes"
+            time mv ~/cassandra /tmp
+            cd /tmp/cassandra/
+            ./pylib/cassandra-cqlsh-tests.sh $(pwd)
+          no_output_timeout: <<parameters.no_output_timeout>>
+      - store_test_results:
+          path: /tmp/cassandra/pylib
+
   run_parallel_junit_tests:
     parameters:
       target:
@@ -2581,7 +2959,12 @@ commands:
             mkdir -p /tmp/results/dtests
             # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
             export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
-            set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+            if [ ! -z "$SPLIT_TESTS" ]; then
+              set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+            else
+              echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
+              (exit 1)
+            fi
       - store_test_results:
           path: /tmp/results
       - store_artifacts:
@@ -2988,7 +3371,7 @@ commands:
         default: ""
     steps:
       - run:
-          name: Run repeated Python dtest
+          name: Run repeated Python DTests
           no_output_timeout: 15m
           command: |
             if [ "<<parameters.tests>>" == "<nil>" ]; then

.circleci/config-2_1.yml.high_res.patch

@@ -1,6 +1,6 @@
---- config-2_1.yml	2022-10-25 11:32:05.718970156 +0100
-+++ config-2_1.yml.HIGHRES	2022-10-25 11:32:31.360769127 +0100
-@@ -149,14 +149,14 @@
+--- config-2_1.yml	2022-11-15 12:54:54.000000000 -0500
++++ config-2_1.yml.HIGHRES	2022-11-15 12:56:26.000000000 -0500
+@@ -158,14 +158,14 @@
  j8_par_executor: &j8_par_executor
    executor:
      name: java8-executor
@@ -19,7 +19,7 @@
  
  j8_small_executor: &j8_small_executor
    executor:
-@@ -167,68 +167,74 @@
+@@ -176,26 +176,26 @@
  j8_medium_par_executor: &j8_medium_par_executor
    executor:
      name: java8-executor
@@ -43,23 +43,23 @@
 +    exec_resource_class: xlarge
 +  parallelism: 100
  
--j11_small_par_executor: &j11_small_par_executor
-+j11_small_executor: &j11_small_executor
+ j11_small_par_executor: &j11_small_par_executor
    executor:
      name: java11-executor
 -    #exec_resource_class: xlarge
-+    exec_resource_class: medium
-   parallelism: 1
- 
--j11_small_executor: &j11_small_executor
-+j11_small_par_executor: &j11_small_par_executor
-   executor:
-     name: java11-executor
--    #exec_resource_class: medium
 -  parallelism: 1
 +    exec_resource_class: xlarge
 +  parallelism: 5
  
+ j11_small_executor: &j11_small_executor
+   executor:
+@@ -207,43 +207,49 @@
+   executor:
+     name: java11-executor
+     #exec_resource_class: xlarge
+-  parallelism: 1
++  parallelism: 2
+ 
  j11_seq_executor: &j11_seq_executor
    executor:
      name: java11-executor
@@ -111,3 +111,39 @@
  
  j8_separate_jobs: &j8_separate_jobs
    jobs:
+@@ -2155,7 +2161,7 @@
+           python_version: '3.8'
+ 
+   j8_dtests_large_vnode:
+-    <<: *j8_par_executor
++    <<: *j8_seq_executor
+     steps:
+       - attach_workspace:
+           at: /home/cassandra
+@@ -2169,7 +2175,7 @@
+           pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+ 
+   j8_dtests_large:
+-    <<: *j8_par_executor
++    <<: *j8_seq_executor
+     steps:
+       - attach_workspace:
+           at: /home/cassandra
+@@ -2183,7 +2189,7 @@
+           pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+ 
+   j11_dtests_large_vnode:
+-    <<: *j11_par_executor
++    <<: *j11_seq_executor
+     steps:
+       - attach_workspace:
+           at: /home/cassandra
+@@ -2197,7 +2203,7 @@
+           pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+ 
+   j11_dtests_large:
+-    <<: *j11_par_executor
++    <<: *j11_seq_executor
+     steps:
+       - attach_workspace:
+           at: /home/cassandra

.circleci/config-2_1.yml.mid_res.patch

@@ -1,6 +1,6 @@
---- config-2_1.yml	2022-11-09 14:47:20.000000000 -0500
-+++ config-2_1.yml.MIDRES	2022-11-09 14:55:10.000000000 -0500
-@@ -149,14 +149,14 @@
+--- config-2_1.yml	2022-11-15 12:54:54.000000000 -0500
++++ config-2_1.yml.MIDRES	2022-11-15 12:56:26.000000000 -0500
+@@ -158,14 +158,14 @@
  j8_par_executor: &j8_par_executor
    executor:
      name: java8-executor
@@ -19,7 +19,7 @@
  
  j8_small_executor: &j8_small_executor
    executor:
-@@ -164,29 +164,41 @@
+@@ -173,29 +173,41 @@
      exec_resource_class: medium
    parallelism: 1
  
@@ -68,16 +68,21 @@
  
  j11_small_executor: &j11_small_executor
    executor:
-@@ -194,41 +206,53 @@
-     #exec_resource_class: medium
-   parallelism: 1
- 
+@@ -206,44 +218,56 @@
+ j11_medium_par_executor: &j11_medium_par_executor
+   executor:
+     name: java11-executor
+-    #exec_resource_class: xlarge
+-  parallelism: 1
++    exec_resource_class: large
++  parallelism: 4
++
 +j11_large_par_executor: &j11_large_par_executor
 +  executor:
 +    name: java11-executor
 +    exec_resource_class: large
 +  parallelism: 50
-+
+ 
  j11_seq_executor: &j11_seq_executor
    executor:
      name: java11-executor
@@ -129,7 +134,7 @@
  
  j8_separate_jobs: &j8_separate_jobs
    jobs:
-@@ -1612,7 +1636,7 @@
+@@ -1856,7 +1880,7 @@
            target: testclasslist-system-keyspace-directory
  
    j8_dtests_vnode:
@@ -138,7 +143,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1626,7 +1650,7 @@
+@@ -1870,7 +1894,7 @@
            pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
  
    j8_dtests_offheap:
@@ -147,7 +152,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1640,7 +1664,7 @@
+@@ -1884,7 +1908,7 @@
            pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
  
    j11_dtests_vnode:
@@ -156,7 +161,7 @@
      steps:
      - attach_workspace:
          at: /home/cassandra
-@@ -1655,7 +1679,7 @@
+@@ -1899,7 +1923,7 @@
          pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
  
    j11_dtests_offheap:
@@ -165,7 +170,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1670,7 +1694,7 @@
+@@ -1914,7 +1938,7 @@
            pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
  
    j8_dtests:
@@ -174,7 +179,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1684,7 +1708,7 @@
+@@ -1928,7 +1952,7 @@
            pytest_extra_args: '--skip-resource-intensive-tests'
  
    j11_dtests:
@@ -183,7 +188,7 @@
      steps:
      - attach_workspace:
          at: /home/cassandra
-@@ -1699,7 +1723,7 @@
+@@ -1943,7 +1967,7 @@
          pytest_extra_args: '--skip-resource-intensive-tests'
  
    j8_upgrade_dtests:
@@ -192,7 +197,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1713,7 +1737,7 @@
+@@ -1957,7 +1981,7 @@
            pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
  
    j8_cqlsh_dtests_py3_vnode:
@@ -201,7 +206,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1728,7 +1752,7 @@
+@@ -1972,7 +1996,7 @@
            extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
  
    j8_cqlsh_dtests_py3_offheap:
@@ -210,7 +215,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1743,7 +1767,7 @@
+@@ -1987,7 +2011,7 @@
            extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
  
    j8_cqlsh_dtests_py38_vnode:
@@ -219,7 +224,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1761,7 +1785,7 @@
+@@ -2005,7 +2029,7 @@
            python_version: '3.8'
  
    j8_cqlsh_dtests_py38_offheap:
@@ -228,7 +233,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1779,7 +1803,7 @@
+@@ -2023,7 +2047,7 @@
            python_version: '3.8'
  
    j8_cqlsh_dtests_py3:
@@ -237,7 +242,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1794,7 +1818,7 @@
+@@ -2038,7 +2062,7 @@
            extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
  
    j8_cqlsh_dtests_py38:
@@ -246,7 +251,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1812,7 +1836,7 @@
+@@ -2056,7 +2080,7 @@
            python_version: '3.8'
  
    j11_cqlsh_dtests_py3_vnode:
@@ -255,7 +260,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1827,7 +1851,7 @@
+@@ -2071,7 +2095,7 @@
            extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
  
    j11_cqlsh_dtests_py3_offheap:
@@ -264,7 +269,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1842,7 +1866,7 @@
+@@ -2086,7 +2110,7 @@
            extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
  
    j11_cqlsh_dtests_py38_vnode:
@@ -273,7 +278,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1860,7 +1884,7 @@
+@@ -2104,7 +2128,7 @@
            python_version: '3.8'
  
    j11_cqlsh_dtests_py38_offheap:
@@ -282,7 +287,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1878,7 +1902,7 @@
+@@ -2122,7 +2146,7 @@
            python_version: '3.8'
  
    j11_cqlsh_dtests_py3:
@@ -291,7 +296,7 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
-@@ -1893,7 +1917,7 @@
+@@ -2137,7 +2161,7 @@
            extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
  
    j11_cqlsh_dtests_py38:
@@ -300,3 +305,39 @@
      steps:
        - attach_workspace:
            at: /home/cassandra
+@@ -2155,7 +2179,7 @@
+           python_version: '3.8'
+ 
+   j8_dtests_large_vnode:
+-    <<: *j8_par_executor
++    <<: *j8_medium_par_executor
+     steps:
+       - attach_workspace:
+           at: /home/cassandra
+@@ -2169,7 +2193,7 @@
+           pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+ 
+   j8_dtests_large:
+-    <<: *j8_par_executor
++    <<: *j8_medium_par_executor
+     steps:
+       - attach_workspace:
+           at: /home/cassandra
+@@ -2183,7 +2207,7 @@
+           pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
+ 
+   j11_dtests_large_vnode:
+-    <<: *j11_par_executor
++    <<: *j11_medium_par_executor
+     steps:
+       - attach_workspace:
+           at: /home/cassandra
+@@ -2197,7 +2221,7 @@
+           pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
+ 
+   j11_dtests_large:
+-    <<: *j11_par_executor
++    <<: *j11_medium_par_executor
+     steps:
+       - attach_workspace:
+           at: /home/cassandra

.circleci/generate.sh

@@ -56,6 +56,8 @@ print_help()
   echo "                   -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500"
   echo "                   -e REPEATED_DTESTS=cdc_test.py cqlsh_tests/test_cqlsh.py::TestCqlshSmoke"
   echo "                   -e REPEATED_DTESTS_COUNT=500"
+  echo "                   -e REPEATED_LARGE_DTESTS=consistency_test.py::TestAvailability::test_network_topology_strategy"
+  echo "                   -e REPEATED_LARGE_DTESTS=100"
   echo "                   -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py upgrade_tests/paging_test.py"
   echo "                   -e REPEATED_UPGRADE_DTESTS_COUNT=25"
   echo "                   -e REPEATED_ANT_TEST_TARGET=testsome"
@@ -87,7 +89,7 @@ while getopts "e:almhf" opt; do
           ;;
       h ) highres=true
           ;;
-      e ) if (!($has_env_vars)); then
+      e ) if (! ($has_env_vars)); then
             env_vars="$OPTARG"
           else
             env_vars="$env_vars|$OPTARG"
@@ -128,6 +130,8 @@ if $has_env_vars && $check_env_vars; then
        [ "$key" != "REPEATED_JVM_UPGRADE_DTESTS_COUNT" ]  &&
        [ "$key" != "REPEATED_DTESTS" ] &&
        [ "$key" != "REPEATED_DTESTS_COUNT" ] &&
+       [ "$key" != "REPEATED_LARGE_DTESTS" ] &&
+       [ "$key" != "REPEATED_LARGE_DTESTS_COUNT" ] &&
        [ "$key" != "REPEATED_UPGRADE_DTESTS" ] &&
        [ "$key" != "REPEATED_UPGRADE_DTESTS_COUNT" ] &&
        [ "$key" != "REPEATED_ANT_TEST_TARGET" ] &&
@@ -187,13 +191,13 @@ elif $all; then
   # copy lower into config.yml to make sure this gets updated
   cp $BASEDIR/config.yml.LOWRES $BASEDIR/config.yml
 
-elif (!($has_env_vars)); then
+elif (! ($has_env_vars)); then
   print_help
   exit 0
 fi
 
 # add new or modified tests to the sets of tests to be repeated
-if (!($all)); then
+if (! ($all)); then
   add_diff_tests ()
   {
     dir="${BASEDIR}/../${2}"
@@ -307,6 +311,12 @@ delete_repeated_jobs()
     delete_job "$1" "j11_dtests_vnode_repeat"
     delete_job "$1" "j11_dtests_offheap_repeat"
   fi
+  if (! (echo "$env_vars" | grep -q "REPEATED_LARGE_DTESTS=")); then
+    delete_job "$1" "j8_dtests_large_repeat"
+    delete_job "$1" "j8_dtests_large_vnode_repeat"
+    delete_job "$1" "j11_dtests_large_repeat"
+    delete_job "$1" "j11_dtests_large_vnode_repeat"
+  fi
   if (! (echo "$env_vars" | grep -q "REPEATED_UPGRADE_DTESTS=")); then
     delete_job "$1" "j8_upgrade_dtests_repeat"
   fi

.circleci/readme.md

@@ -78,6 +78,7 @@ generate.sh -m \
   -e REPEATED_JVM_DTESTS_COUNT=500 \
   -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500 \
   -e REPEATED_DTESTS_COUNT=500 \
+  -e REPEATED_LARGE_DTESTS_COUNT=100 \
   -e REPEATED_UPGRADE_DTESTS_COUNT=25 \
   -e REPEATED_ANT_TEST_COUNT=500
 ```
@@ -95,6 +96,7 @@ generate.sh -m \
   -e REPEATED_UTESTS_STRESS=org.apache.cassandra.stress.generate.DistributionGaussianTest \
   -e REPEATED_SIMULATOR_DTESTS=org.apache.cassandra.simulator.test.TrivialSimulationTest \
   -e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \
+  -e REPEATED_LARGE_DTESTS=consistency_test.py::TestAvailability::test_network_topology_strategy \
   -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest#testPaging \
   -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py \
   -e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest
@@ -125,6 +127,8 @@ generate.sh -m \
   -e REPEATED_SIMULATOR_DTESTS_COUNT=500 \
   -e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \
   -e REPEATED_DTESTS_COUNT=500 \
+  -e REPEATED_LARGE_DTESTS=consistency_test.py,materialized_views_test.py \
+  -e REPEATED_LARGE_DTESTS=100 \
   -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest#testPaging \
   -e REPEATED_JVM_DTESTS_COUNT=500 \
   -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py \

pylib/cassandra-cqlsh-tests.sh

@@ -39,7 +39,7 @@ export CASS_DRIVER_NO_CYTHON=true
 export CCM_MAX_HEAP_SIZE="2048M"
 export CCM_HEAP_NEWSIZE="200M"
 export CCM_CONFIG_DIR=${WORKSPACE}/.ccm
-export NUM_TOKENS="32"
+export NUM_TOKENS="16"
 export CASSANDRA_DIR=${WORKSPACE}
 export TESTSUITE_NAME="cqlshlib.${PYTHON_VERSION}"