natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch Java and Python upgrade tests from running on JDK8 to JDK11 in CircleCI 

patch by Ekaterina Dimitrova; reviewed by Berenguer Blasi for CASSANDRA-18285
This commit is contained in:
Ekaterina Dimitrova 2023-06-05 21:47:09 -04:00
parent 43d90928a8
commit 1d961679a0
12 changed files with 4730 additions and 1598 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

969
.circleci/config.yml
View File

File diff suppressed because it is too large Load Diff

969
.circleci/config.yml.FREE
View File

File diff suppressed because it is too large Load Diff

971
.circleci/config.yml.PAID
View File

File diff suppressed because it is too large Load Diff

978
.circleci/config_11_and_17.yml
View File

File diff suppressed because it is too large Load Diff

978
.circleci/config_11_and_17.yml.FREE
View File

File diff suppressed because it is too large Load Diff

978
.circleci/config_11_and_17.yml.PAID
View File

File diff suppressed because it is too large Load Diff

146
.circleci/config_template.yml
View File

@ -224,14 +224,14 @@ j8_repeated_dtest_executor: &j8_repeated_dtest_executor
name: java8-executor
parallelism: 4
j8_repeated_upgrade_dtest_executor: &j8_repeated_upgrade_dtest_executor
j11_repeated_upgrade_dtest_executor: &j11_repeated_upgrade_dtest_executor
executor:
name: java8-executor
name: java11-executor
parallelism: 4
j8_repeated_jvm_upgrade_dtest_executor: &j8_repeated_jvm_upgrade_dtest_executor
j11_repeated_jvm_upgrade_dtest_executor: &j11_repeated_jvm_upgrade_dtest_executor
executor:
name: java8-executor
name: java11-executor
parallelism: 4
j11_repeated_utest_executor: &j11_repeated_utest_executor
@ -550,24 +550,6 @@ j8_separate_jobs: &j8_separate_jobs
requires:
- start_j11_utests_system_keyspace_directory_repeat
- j8_build
- start_j8_dtest_jars_build:
type: approval
- j8_dtest_jars_build:
requires:
- j8_build
- start_j8_dtest_jars_build
- start_jvm_upgrade_dtests:
type: approval
- j8_jvm_upgrade_dtests:
requires:
- start_jvm_upgrade_dtests
- j8_dtest_jars_build
- start_jvm_upgrade_dtests_repeat:
type: approval
- j8_jvm_upgrade_dtests_repeat:
requires:
- start_jvm_upgrade_dtests_repeat
- j8_dtest_jars_build
# Python DTests
- start_j8_dtests:
type: approval
@ -669,13 +651,6 @@ j8_separate_jobs: &j8_separate_jobs
requires:
- start_j11_dtests_large_vnode_repeat
- j8_build
# Java 8 upgrade tests
- start_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
requires:
- start_upgrade_dtests
- j8_build
# Java 8 cqlsh dtests
- start_j8_cqlsh_tests:
type: approval
@ -800,13 +775,6 @@ j8_separate_jobs: &j8_separate_jobs
requires:
- start_j11_dtests_vnode_repeat
- j8_build
# Repeated Python upgrade dtest
- start_j8_upgrade_dtests_repeat:
type: approval
- j8_upgrade_dtests_repeat:
requires:
- start_j8_upgrade_dtests_repeat
- j8_build
j8_pre-commit_jobs: &j8_pre-commit_jobs
jobs:
@ -1008,18 +976,6 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs
requires:
- start_utests_system_keyspace_directory
- j8_build
- start_jvm_upgrade_dtests:
type: approval
- j8_dtest_jars_build:
requires:
- j8_build
- start_jvm_upgrade_dtests
- j8_jvm_upgrade_dtests:
requires:
- j8_dtest_jars_build
- j8_jvm_upgrade_dtests_repeat:
requires:
- j8_dtest_jars_build
# Python DTests
- j8_dtests:
requires:
@ -1103,17 +1059,6 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs
requires:
- start_j11_dtests_large
- j8_build
# Java 8 upgrade tests (on request)
- start_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
requires:
- j8_build
- start_upgrade_dtests
- j8_upgrade_dtests_repeat:
requires:
- j8_build
- start_upgrade_dtests
# Java 8 cqlsh dtests
- j8_cqlsh_dtests_py3:
requires:
@ -1435,6 +1380,32 @@ j11_separate_jobs: &j11_separate_jobs
requires:
- start_j11_utests_system_keyspace_directory_repeat
- j11_build
# Java upgrade tests
- start_j11_dtest_jars_build:
type: approval
- j11_dtest_jars_build:
requires:
- j11_build
- start_j11_dtest_jars_build
- start_jvm_upgrade_dtests:
type: approval
- j11_jvm_upgrade_dtests:
requires:
- start_jvm_upgrade_dtests
- j11_dtest_jars_build
- start_jvm_upgrade_dtests_repeat:
type: approval
- j11_jvm_upgrade_dtests_repeat:
requires:
- start_jvm_upgrade_dtests_repeat
- j11_dtest_jars_build
# Java 11 upgrade tests
- start_j11_upgrade_dtests:
type: approval
- j11_upgrade_dtests:
requires:
- start_j11_upgrade_dtests
- j11_build
# Java 11 repeated utest
- start_j11_repeated_ant_test:
type: approval
@ -1455,6 +1426,13 @@ j11_separate_jobs: &j11_separate_jobs
requires:
- start_j11_dtests_vnode_repeat
- j11_build
# Repeated Python upgrade dtest
- start_j11_upgrade_dtests_repeat:
type: approval
- j11_upgrade_dtests_repeat:
requires:
- start_j11_upgrade_dtests_repeat
- j11_build
j11_pre-commit_jobs: &j11_pre-commit_jobs
jobs:
@ -1644,6 +1622,30 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs
requires:
- start_utests_system_keyspace_directory
- j11_build
# Java upgrade tests
- start_jvm_upgrade_dtests:
type: approval
- j11_dtest_jars_build:
requires:
- j11_build
- start_jvm_upgrade_dtests
- j11_jvm_upgrade_dtests:
requires:
- j11_dtest_jars_build
- j11_jvm_upgrade_dtests_repeat:
requires:
- j11_dtest_jars_build
# Java 11 upgrade tests (on request)
- start_j11_upgrade_dtests:
type: approval
- j11_upgrade_dtests:
requires:
- j11_build
- start_j11_upgrade_dtests
- j11_upgrade_dtests_repeat:
requires:
- j11_build
- start_j11_upgrade_dtests
workflows:
version: 2
@ -1706,8 +1708,8 @@ jobs:
executor: java11-executor
<<: *build_common
j8_dtest_jars_build:
executor: java8-executor
j11_dtest_jars_build:
executor: java11-executor
parallelism: 1
steps:
- attach_workspace:
@ -1809,8 +1811,8 @@ jobs:
target: "testclasslist"
arguments: "-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
j8_jvm_upgrade_dtests:
<<: *j8_medium_par_executor
j11_jvm_upgrade_dtests:
<<: *j11_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@ -2084,18 +2086,18 @@ jobs:
file_tag: j11_without_vnodes
pytest_extra_args: '--skip-resource-intensive-tests'
j8_upgrade_dtests:
<<: *j8_par_executor
j11_upgrade_dtests:
<<: *j11_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- create_dtest_containers:
file_tag: j8_upgradetests_without_vnodes
file_tag: j11_upgradetests_without_vnodes
run_dtests_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
- run_dtests:
file_tag: j8_upgradetests_without_vnodes
file_tag: j11_upgradetests_without_vnodes
pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
j8_cqlsh_dtests_py3_vnode:
@ -2636,8 +2638,8 @@ jobs:
- log_environment
- run_simulator_dtests_repeat
j8_jvm_upgrade_dtests_repeat:
<<: *j8_repeated_jvm_upgrade_dtest_executor
j11_jvm_upgrade_dtests_repeat:
<<: *j11_repeated_jvm_upgrade_dtest_executor
steps:
- attach_workspace:
at: /home/cassandra
@ -2836,8 +2838,8 @@ jobs:
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
j8_upgrade_dtests_repeat:
<<: *j8_repeated_upgrade_dtest_executor
j11_upgrade_dtests_repeat:
<<: *j11_repeated_upgrade_dtest_executor
steps:
- attach_workspace:
at: /home/cassandra
@ -2921,7 +2923,7 @@ commands:
cd ~/cassandra
mkdir ~/dtest_jars
git remote add apache https://github.com/apache/cassandra.git
for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 cassandra-4.1 trunk; do
for branch in cassandra-4.0 cassandra-4.1 trunk; do
# check out the correct cassandra version:
git remote set-branches --add apache '$branch'
git fetch --depth 1 apache $branch

86
.circleci/config_template.yml.PAID.patch
View File

@ -1,6 +1,6 @@
--- a/config_template.yml
+++ b/config_template.yml.PAID
@@ -157,14 +157,14 @@ default_env_vars: &default_env_vars
--- config_template.yml 2023-06-06 17:25:41.000000000 -0400
+++ config_template.yml.PAID 2023-06-06 17:28:54.000000000 -0400
@@ -157,14 +157,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
@ -19,7 +19,7 @@
j8_small_executor: &j8_small_executor
executor:
@@ -172,29 +172,41 @@ j8_small_executor: &j8_small_executor
@@ -172,29 +172,41 @@
exec_resource_class: medium
parallelism: 1
@ -29,9 +29,9 @@
+ exec_resource_class: large
+ parallelism: 50
+
+j8_very_large_par_executor: &j8_very_large_par_executor
+j11_very_large_par_executor: &j11_very_large_par_executor
+ executor:
+ name: java8-executor
+ name: java11-executor
+ exec_resource_class: large
+ parallelism: 100
+
@ -68,7 +68,7 @@
j11_small_executor: &j11_small_executor
executor:
@@ -205,44 +217,56 @@ j11_small_executor: &j11_small_executor
@@ -205,44 +217,56 @@
j11_medium_par_executor: &j11_medium_par_executor
executor:
name: java11-executor
@ -104,16 +104,16 @@
+ exec_resource_class: large
+ parallelism: 25
j8_repeated_upgrade_dtest_executor: &j8_repeated_upgrade_dtest_executor
j11_repeated_upgrade_dtest_executor: &j11_repeated_upgrade_dtest_executor
executor:
name: java8-executor
name: java11-executor
- parallelism: 4
+ exec_resource_class: xlarge
+ parallelism: 25
j8_repeated_jvm_upgrade_dtest_executor: &j8_repeated_jvm_upgrade_dtest_executor
j11_repeated_jvm_upgrade_dtest_executor: &j11_repeated_jvm_upgrade_dtest_executor
executor:
name: java8-executor
name: java11-executor
- parallelism: 4
+ exec_resource_class: large
+ parallelism: 25
@ -134,7 +134,7 @@
j8_separate_jobs: &j8_separate_jobs
jobs:
@@ -1998,7 +2022,7 @@ jobs:
@@ -2000,7 +2024,7 @@
target: testclasslist-system-keyspace-directory
j8_dtests_vnode:
@ -143,7 +143,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2012,7 +2036,7 @@ jobs:
@@ -2014,7 +2038,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j8_dtests_offheap:
@ -152,7 +152,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2026,7 +2050,7 @@ jobs:
@@ -2028,7 +2052,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j11_dtests_vnode:
@ -161,7 +161,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2041,7 +2065,7 @@ jobs:
@@ -2043,7 +2067,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j11_dtests_offheap:
@ -170,7 +170,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2056,7 +2080,7 @@ jobs:
@@ -2058,7 +2082,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j8_dtests:
@ -179,7 +179,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2070,7 +2094,7 @@ jobs:
@@ -2072,7 +2096,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j11_dtests:
@ -188,16 +188,16 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2085,7 +2109,7 @@ jobs:
@@ -2087,7 +2111,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j8_upgrade_dtests:
- <<: *j8_par_executor
+ <<: *j8_very_large_par_executor
j11_upgrade_dtests:
- <<: *j11_par_executor
+ <<: *j11_very_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2099,7 +2123,7 @@ jobs:
@@ -2101,7 +2125,7 @@
pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
j8_cqlsh_dtests_py3_vnode:
@ -206,7 +206,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2114,7 +2138,7 @@ jobs:
@@ -2116,7 +2140,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py3_offheap:
@ -215,7 +215,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2129,7 +2153,7 @@ jobs:
@@ -2131,7 +2155,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py38_vnode:
@ -224,7 +224,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2147,7 +2171,7 @@ jobs:
@@ -2149,7 +2173,7 @@
python_version: '3.8'
j8_cqlsh_dtests_py311_vnode:
@ -233,7 +233,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2165,7 +2189,7 @@ jobs:
@@ -2167,7 +2191,7 @@
python_version: '3.11'
j8_cqlsh_dtests_py38_offheap:
@ -242,7 +242,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2183,7 +2207,7 @@ jobs:
@@ -2185,7 +2209,7 @@
python_version: '3.8'
j8_cqlsh_dtests_py311_offheap:
@ -251,7 +251,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2201,7 +2225,7 @@ jobs:
@@ -2203,7 +2227,7 @@
python_version: '3.11'
j8_cqlsh_dtests_py3:
@ -260,7 +260,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2216,7 +2240,7 @@ jobs:
@@ -2218,7 +2242,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py38:
@ -269,7 +269,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2234,7 +2258,7 @@ jobs:
@@ -2236,7 +2260,7 @@
python_version: '3.8'
j8_cqlsh_dtests_py311:
@ -278,7 +278,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2252,7 +2276,7 @@ jobs:
@@ -2254,7 +2278,7 @@
python_version: '3.11'
j11_cqlsh_dtests_py3_vnode:
@ -287,7 +287,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2267,7 +2291,7 @@ jobs:
@@ -2269,7 +2293,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py3_offheap:
@ -296,7 +296,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2282,7 +2306,7 @@ jobs:
@@ -2284,7 +2308,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38_vnode:
@ -305,7 +305,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2300,7 +2324,7 @@ jobs:
@@ -2302,7 +2326,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311_vnode:
@ -314,7 +314,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2318,7 +2342,7 @@ jobs:
@@ -2320,7 +2344,7 @@
python_version: '3.11'
j11_cqlsh_dtests_py38_offheap:
@ -323,7 +323,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2336,7 +2360,7 @@ jobs:
@@ -2338,7 +2362,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311_offheap:
@ -332,7 +332,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2354,7 +2378,7 @@ jobs:
@@ -2356,7 +2380,7 @@
python_version: '3.11'
j11_cqlsh_dtests_py3:
@ -341,7 +341,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2369,7 +2393,7 @@ jobs:
@@ -2371,7 +2395,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38:
@ -350,7 +350,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2387,7 +2411,7 @@ jobs:
@@ -2389,7 +2413,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311:
@ -359,7 +359,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2405,7 +2429,7 @@ jobs:
@@ -2407,7 +2431,7 @@
python_version: '3.11'
j8_dtests_large_vnode:
@ -368,7 +368,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2419,7 +2443,7 @@ jobs:
@@ -2421,7 +2445,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_dtests_large:
@ -377,7 +377,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2433,7 +2457,7 @@ jobs:
@@ -2435,7 +2459,7 @@
pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
j11_dtests_large_vnode:
@ -386,7 +386,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2447,7 +2471,7 @@ jobs:
@@ -2449,7 +2473,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j11_dtests_large:

144
.circleci/config_template_11_and_17.yml
View File

@ -95,6 +95,14 @@ default_env_vars: &default_env_vars
# The number of times that new, modified or manually specified JVM dtests should be run.
REPEATED_JVM_DTESTS_COUNT: 500
# Comma-separated list of tests that should be included in the repeated run for JVM upgrade dtests,
# in addition to automatically detected new and modified tests. For example:
# REPEATED_JVM_UPGRADE_DTESTS: org.apache.cassandra.distributed.upgrade.GroupByTest
# REPEATED_JVM_UPGRADE_DTESTS: org.apache.cassandra.distributed.upgrade.GroupByTest#testReads
REPEATED_JVM_UPGRADE_DTESTS:
# The number of times that new, modified or manually specified JVM upgrade dtests should be run.
REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
# Comma-separated list of Python dtests that should be repeatedly run, for example:
# REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py
# REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke
@ -113,6 +121,14 @@ default_env_vars: &default_env_vars
# The number of times that the manually specified Python large dtests should be run.
REPEATED_LARGE_DTESTS_COUNT: 100
# Comma-separated list of Python dtests that should be repeatedly run, for example:
# REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py
# REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py::TestCQLNodes2RF1_Upgrade_current_4_0_x_To_indev_4_0_x
# REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py,upgrade_tests/paging_test.py
REPEATED_UPGRADE_DTESTS:
# The number of times that the manually specified Python upgrade dtests should be run.
REPEATED_UPGRADE_DTESTS_COUNT: 25
# The Ant test target to run, for example:
# REPEATED_ANT_TEST_TARGET: testsome
# REPEATED_ANT_TEST_TARGET: test-jvm-dtest-some
@ -199,6 +215,11 @@ j17_seq_executor: &j17_seq_executor
#exec_resource_class: xlarge
parallelism: 1 # sequential, single container tests: no parallelism benefits
j11_repeated_jvm_upgrade_dtest_executor: &j11_repeated_jvm_upgrade_dtest_executor
executor:
name: java11-executor
parallelism: 4
j11_repeated_utest_executor: &j11_repeated_utest_executor
executor:
name: java11-executor
@ -219,6 +240,11 @@ j17_repeated_dtest_executor: &j17_repeated_dtest_executor
name: java17-executor
parallelism: 4
j11_repeated_upgrade_dtest_executor: &j11_repeated_upgrade_dtest_executor
executor:
name: java11-executor
parallelism: 4
j11_separate_jobs: &j11_separate_jobs
jobs:
- start_j11_build:
@ -525,6 +551,25 @@ j11_separate_jobs: &j11_separate_jobs
requires:
- start_j17_utests_system_keyspace_directory_repeat
- j11_build
# Java upgrade tests
- start_j11_dtest_jars_build:
type: approval
- j11_dtest_jars_build:
requires:
- j11_build
- start_j11_dtest_jars_build
- start_jvm_upgrade_dtests:
type: approval
- j11_jvm_upgrade_dtests:
requires:
- start_jvm_upgrade_dtests
- j11_dtest_jars_build
- start_jvm_upgrade_dtests_repeat:
type: approval
- j11_jvm_upgrade_dtests_repeat:
requires:
- start_jvm_upgrade_dtests_repeat
- j11_dtest_jars_build
# Python DTests
- start_j11_dtests:
type: approval
@ -710,6 +755,13 @@ j11_separate_jobs: &j11_separate_jobs
requires:
- start_j17_cqlsh_tests_offheap
- j11_build
# Java 11 upgrade tests
- start_j11_upgrade_dtests:
type: approval
- j11_upgrade_dtests:
requires:
- start_j11_upgrade_dtests
- j11_build
# Java 11 repeated utest
- start_j11_repeated_ant_test:
type: approval
@ -750,6 +802,13 @@ j11_separate_jobs: &j11_separate_jobs
requires:
- start_j17_dtests_vnode_repeat
- j11_build
# Repeated Python upgrade dtest
- start_j11_upgrade_dtests_repeat:
type: approval
- j11_upgrade_dtests_repeat:
requires:
- start_j11_upgrade_dtests_repeat
- j11_build
j11_pre-commit_jobs: &j11_pre-commit_jobs
jobs:
@ -951,6 +1010,19 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs
requires:
- start_utests_system_keyspace_directory
- j11_build
# Java upgrade tests
- start_jvm_upgrade_dtests:
type: approval
- j11_dtest_jars_build:
requires:
- j11_build
- start_jvm_upgrade_dtests
- j11_jvm_upgrade_dtests:
requires:
- j11_dtest_jars_build
- j11_jvm_upgrade_dtests_repeat:
requires:
- j11_dtest_jars_build
# Python DTests
- j11_dtests:
requires:
@ -1102,6 +1174,17 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs
requires:
- start_j17_cqlsh-dtests-offheap
- j11_build
# Java 11 upgrade tests (on request)
- start_j11_upgrade_dtests:
type: approval
- j11_upgrade_dtests:
requires:
- j11_build
- start_j11_upgrade_dtests
- j11_upgrade_dtests_repeat:
requires:
- j11_build
- start_j11_upgrade_dtests
j17_separate_jobs: &j17_separate_jobs
jobs:
@ -1569,6 +1652,7 @@ executors:
<<: *default_env_vars
JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
CASSANDRA_USE_JDK11: true
java17-executor:
parameters:
@ -1687,6 +1771,19 @@ jobs:
classlistprefix: distributed
target: "testclasslist"
j11_jvm_upgrade_dtests:
<<: *j11_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- create_junit_containers:
classlistprefix: distributed
extra_filters: "| grep upgrade"
- log_environment
- run_parallel_junit_tests:
classlistprefix: distributed
target: "testclasslist"
j17_jvm_dtests_vnode:
<<: *j17_small_par_executor
steps:
@ -1963,6 +2060,20 @@ jobs:
file_tag: j17_without_vnodes
pytest_extra_args: '--skip-resource-intensive-tests'
j11_upgrade_dtests:
<<: *j11_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- create_dtest_containers:
file_tag: j11_upgradetests_without_vnodes
run_dtests_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
- run_dtests:
file_tag: j11_upgradetests_without_vnodes
pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
j11_cqlsh_dtests_py3_vnode:
<<: *j11_par_executor
steps:
@ -2469,6 +2580,14 @@ jobs:
- log_environment
- run_utests_stress_repeat
j11_jvm_upgrade_dtests_repeat:
<<: *j11_repeated_jvm_upgrade_dtest_executor
steps:
- attach_workspace:
at: /home/cassandra
- log_environment
- run_jvm_upgrade_dtests_repeat
j11_jvm_dtests_repeat:
<<: *j11_repeated_utest_executor
steps:
@ -2685,6 +2804,20 @@ jobs:
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
j11_upgrade_dtests_repeat:
<<: *j11_repeated_upgrade_dtest_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- run_repeated_dtest:
tests: ${REPEATED_UPGRADE_DTESTS}
vnodes: "false"
upgrade: "true"
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
count: ${REPEATED_UPGRADE_DTESTS_COUNT}
commands:
log_environment:
steps:
@ -2756,7 +2889,7 @@ commands:
cd ~/cassandra
mkdir ~/dtest_jars
git remote add apache https://github.com/apache/cassandra.git
for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 cassandra-4.1 trunk; do
for branch in cassandra-4.0 cassandra-4.1 trunk; do
# check out the correct cassandra version:
git remote set-branches --add apache '$branch'
git fetch --depth 1 apache $branch
@ -3181,6 +3314,15 @@ commands:
vnodes: true
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
run_jvm_upgrade_dtests_repeat:
steps:
- run_repeated_utests:
target: test-jvm-dtest-some
tests: ${REPEATED_JVM_UPGRADE_DTESTS}
count: ${REPEATED_JVM_UPGRADE_DTESTS_COUNT}
vnodes: false
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
run_repeated_utests:
parameters:
target:

89
.circleci/config_template_11_and_17.yml.PAID.patch
View File

@ -1,6 +1,6 @@
--- a/config_template_11_and_17.yml
+++ b/config_template_11_and_17.yml.PAID
@@ -141,14 +141,14 @@ default_env_vars: &default_env_vars
--- config_template_11_and_17.yml 2023-06-09 19:10:52.000000000 -0400
+++ config_template_11_and_17.yml.PAID 2023-06-09 19:34:32.000000000 -0400
@@ -157,14 +157,14 @@
j11_par_executor: &j11_par_executor
executor:
name: java11-executor
@ -19,7 +19,7 @@
j11_small_executor: &j11_small_executor
executor:
@@ -156,30 +156,41 @@ j11_small_executor: &j11_small_executor
@@ -172,30 +172,41 @@
exec_resource_class: medium
parallelism: 1
@ -68,7 +68,7 @@
j17_small_executor: &j17_small_executor
executor:
@@ -190,34 +201,44 @@ j17_small_executor: &j17_small_executor
@@ -206,44 +217,56 @@
j17_medium_par_executor: &j17_medium_par_executor
executor:
name: java17-executor
@ -90,6 +90,13 @@
+ exec_resource_class: medium
parallelism: 1 # sequential, single container tests: no parallelism benefits
j11_repeated_jvm_upgrade_dtest_executor: &j11_repeated_jvm_upgrade_dtest_executor
executor:
name: java11-executor
- parallelism: 4
+ exec_resource_class: large
+ parallelism: 25
j11_repeated_utest_executor: &j11_repeated_utest_executor
executor:
name: java11-executor
@ -116,11 +123,18 @@
name: java17-executor
- parallelism: 4
+ exec_resource_class: large
+ parallelism: 25
j11_repeated_upgrade_dtest_executor: &j11_repeated_upgrade_dtest_executor
executor:
name: java11-executor
- parallelism: 4
+ exec_resource_class: xlarge
+ parallelism: 25
j11_separate_jobs: &j11_separate_jobs
jobs:
@@ -1877,7 +1898,7 @@ jobs:
@@ -1974,7 +1997,7 @@
target: testclasslist-system-keyspace-directory
j11_dtests_vnode:
@ -129,7 +143,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -1891,7 +1912,7 @@ jobs:
@@ -1988,7 +2011,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j11_dtests_offheap:
@ -138,7 +152,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -1905,7 +1926,7 @@ jobs:
@@ -2002,7 +2025,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j17_dtests_vnode:
@ -147,7 +161,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -1920,7 +1941,7 @@ jobs:
@@ -2017,7 +2040,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j17_dtests_offheap:
@ -156,7 +170,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -1935,7 +1956,7 @@ jobs:
@@ -2032,7 +2055,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j11_dtests:
@ -165,7 +179,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -1949,7 +1970,7 @@ jobs:
@@ -2046,7 +2069,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j17_dtests:
@ -174,16 +188,25 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -1964,7 +1985,7 @@ jobs:
@@ -2061,7 +2084,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j11_upgrade_dtests:
- <<: *j11_par_executor
+ <<: *j11_very_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2075,7 +2098,7 @@
pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
j11_cqlsh_dtests_py3_vnode:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1979,7 +2000,7 @@ jobs:
@@ -2090,7 +2113,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py3_offheap:
@ -192,7 +215,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -1994,7 +2015,7 @@ jobs:
@@ -2105,7 +2128,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38_vnode:
@ -201,7 +224,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2012,7 +2033,7 @@ jobs:
@@ -2123,7 +2146,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311_vnode:
@ -210,7 +233,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2030,7 +2051,7 @@ jobs:
@@ -2141,7 +2164,7 @@
python_version: '3.11'
j11_cqlsh_dtests_py38_offheap:
@ -219,7 +242,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2048,7 +2069,7 @@ jobs:
@@ -2159,7 +2182,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311_offheap:
@ -228,7 +251,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2066,7 +2087,7 @@ jobs:
@@ -2177,7 +2200,7 @@
python_version: '3.11'
j11_cqlsh_dtests_py3:
@ -237,7 +260,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2081,7 +2102,7 @@ jobs:
@@ -2192,7 +2215,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38:
@ -246,7 +269,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2099,7 +2120,7 @@ jobs:
@@ -2210,7 +2233,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311:
@ -255,7 +278,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2117,7 +2138,7 @@ jobs:
@@ -2228,7 +2251,7 @@
python_version: '3.11'
j17_cqlsh_dtests_py3_vnode:
@ -264,7 +287,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2132,7 +2153,7 @@ jobs:
@@ -2243,7 +2266,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j17_cqlsh_dtests_py3_offheap:
@ -273,7 +296,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2147,7 +2168,7 @@ jobs:
@@ -2258,7 +2281,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j17_cqlsh_dtests_py38_vnode:
@ -282,7 +305,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2165,7 +2186,7 @@ jobs:
@@ -2276,7 +2299,7 @@
python_version: '3.8'
j17_cqlsh_dtests_py311_vnode:
@ -291,7 +314,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2183,7 +2204,7 @@ jobs:
@@ -2294,7 +2317,7 @@
python_version: '3.11'
j17_cqlsh_dtests_py38_offheap:
@ -300,7 +323,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2201,7 +2222,7 @@ jobs:
@@ -2312,7 +2335,7 @@
python_version: '3.8'
j17_cqlsh_dtests_py311_offheap:
@ -309,7 +332,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2219,7 +2240,7 @@ jobs:
@@ -2330,7 +2353,7 @@
python_version: '3.11'
j17_cqlsh_dtests_py3:
@ -318,7 +341,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2234,7 +2255,7 @@ jobs:
@@ -2345,7 +2368,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j17_cqlsh_dtests_py38:
@ -327,7 +350,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2252,7 +2273,7 @@ jobs:
@@ -2363,7 +2386,7 @@
python_version: '3.8'
j17_cqlsh_dtests_py311:
@ -336,7 +359,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2270,7 +2291,7 @@ jobs:
@@ -2381,7 +2404,7 @@
python_version: '3.11'
j11_dtests_large_vnode:
@ -345,7 +368,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2284,7 +2305,7 @@ jobs:
@@ -2395,7 +2418,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j11_dtests_large:
@ -354,7 +377,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2298,7 +2319,7 @@ jobs:
@@ -2409,7 +2432,7 @@
pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
j17_dtests_large_vnode:
@ -363,7 +386,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -2312,7 +2333,7 @@ jobs:
@@ -2423,7 +2446,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j17_dtests_large:

4
.circleci/generate.sh
View File

@ -308,7 +308,7 @@ delete_repeated_jobs()
fi
if (! (echo "$env_vars" | grep -q "REPEATED_JVM_UPGRADE_DTESTS=")); then
delete_job "$1" "start_jvm_upgrade_dtests_repeat"
delete_job "$1" "j8_jvm_upgrade_dtests_repeat"
delete_job "$1" "j11_jvm_upgrade_dtests_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_DTESTS=")); then
delete_job "$1" "j8_dtests_repeat"
@ -325,7 +325,7 @@ delete_repeated_jobs()
delete_job "$1" "j11_dtests_large_vnode_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_UPGRADE_DTESTS=")); then
delete_job "$1" "j8_upgrade_dtests_repeat"
delete_job "$1" "j11_upgrade_dtests_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_ANT_TEST_CLASS=")); then
delete_job "$1" "j8_repeated_ant_test"

16
.circleci/generate_11_and_17.sh
View File

@ -57,10 +57,14 @@ print_help()
echo " -e REPEATED_SIMULATOR_DTESTS_COUNT=500"
echo " -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest"
echo " -e REPEATED_JVM_DTESTS_COUNT=500"
echo " -e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest"
echo " -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500"
echo " -e REPEATED_DTESTS=cdc_test.py cqlsh_tests/test_cqlsh.py::TestCqlshSmoke"
echo " -e REPEATED_DTESTS_COUNT=500"
echo " -e REPEATED_LARGE_DTESTS=replace_address_test.py::TestReplaceAddress::test_replace_stopped_node"
echo " -e REPEATED_LARGE_DTESTS=100"
echo " -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py upgrade_tests/paging_test.py"
echo " -e REPEATED_UPGRADE_DTESTS_COUNT=25"
echo " -e REPEATED_ANT_TEST_TARGET=testsome"
echo " -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest"
echo " -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable"
@ -130,10 +134,14 @@ if $has_env_vars && $check_env_vars; then
[ "$key" != "REPEATED_SIMULATOR_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_JVM_DTESTS" ] &&
[ "$key" != "REPEATED_JVM_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_JVM_UPGRADE_DTESTS" ] &&
[ "$key" != "REPEATED_JVM_UPGRADE_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_DTESTS" ] &&
[ "$key" != "REPEATED_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_LARGE_DTESTS" ] &&
[ "$key" != "REPEATED_LARGE_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_UPGRADE_DTESTS" ] &&
[ "$key" != "REPEATED_UPGRADE_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_ANT_TEST_TARGET" ] &&
[ "$key" != "REPEATED_ANT_TEST_CLASS" ] &&
[ "$key" != "REPEATED_ANT_TEST_METHODS" ] &&
@ -228,6 +236,7 @@ if $detect_changed_tests; then
add_diff_tests "REPEATED_UTESTS_FQLTOOL" "tools/fqltool/test/unit/" "org.apache.cassandra.fqltool"
add_diff_tests "REPEATED_SIMULATOR_DTESTS" "test/simulator/test/" "org.apache.cassandra.simulator.test"
add_diff_tests "REPEATED_JVM_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.test"
add_diff_tests "REPEATED_JVM_UPGRADE_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.upgrade"
fi
# replace environment variables
@ -299,6 +308,10 @@ delete_repeated_jobs()
delete_job "$1" "j17_jvm_dtests_repeat"
delete_job "$1" "j17_jvm_dtests_vnode_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_JVM_UPGRADE_DTESTS=")); then
delete_job "$1" "start_jvm_upgrade_dtests_repeat"
delete_job "$1" "j11_jvm_upgrade_dtests_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_DTESTS=")); then
delete_job "$1" "j11_dtests_repeat"
delete_job "$1" "j11_dtests_vnode_repeat"
@ -313,6 +326,9 @@ delete_repeated_jobs()
delete_job "$1" "j17_dtests_large_repeat"
delete_job "$1" "j17_dtests_large_vnode_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_UPGRADE_DTESTS=")); then
delete_job "$1" "j11_upgrade_dtests_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_ANT_TEST_CLASS=")); then
delete_job "$1" "j11_repeated_ant_test"
delete_job "$1" "j17_repeated_ant_test"