From 0b5248d8c3ef743a8dda42f7be5dadb70e373b51 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Sun, 4 Dec 2022 13:05:19 -0600 Subject: [PATCH 01/10] Accommodate python 3.11 Patch by brandonwilliams; reviewed by bereng for CASSANDRA-18088 --- CHANGES.txt | 1 + pylib/cqlshlib/saferscanner.py | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d51e991e34..8794ae533a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0.8 + * Add support for python 3.11 (CASSANDRA-18088) * Fix formatting of duration in cqlsh (CASSANDRA-18141) * Fix sstable loading of keyspaces named snapshots or backups (CASSANDRA-14013) * Avoid ConcurrentModificationException in STCS/DTCS/TWCS.getSSTables (CASSANDRA-17977) diff --git a/pylib/cqlshlib/saferscanner.py b/pylib/cqlshlib/saferscanner.py index 8949321dff..6d7ba571af 100644 --- a/pylib/cqlshlib/saferscanner.py +++ b/pylib/cqlshlib/saferscanner.py @@ -20,7 +20,10 @@ import re import six -from sre_constants import BRANCH, SUBPATTERN, GROUPREF, GROUPREF_IGNORE, GROUPREF_EXISTS +try: + from sre_constants import BRANCH, SUBPATTERN, GROUPREF, GROUPREF_IGNORE, GROUPREF_EXISTS +except ImportError: + from re._constants import BRANCH, SUBPATTERN, GROUPREF, GROUPREF_IGNORE, GROUPREF_EXISTS from sys import version_info @@ -99,5 +102,24 @@ class Py38SaferScanner(SaferScannerBase): self.scanner = re.sre_compile.compile(p) +class Py311SaferScanner(SaferScannerBase): + + def __init__(self, lexicon, flags=0): + self.lexicon = lexicon + p = [] + s = re._parser.State() + s.flags = flags + for phrase, action in lexicon: + gid = s.opengroup() + p.append(re._parser.SubPattern(s, [(SUBPATTERN, (gid, 0, 0, re._parser.parse(phrase, flags))), ])) + s.closegroup(gid, p[-1]) + p = re._parser.SubPattern(s, [(BRANCH, (None, p))]) + self.p = p + self.scanner = re._compiler.compile(p) + + SaferScanner = Py36SaferScanner if six.PY3 else Py2SaferScanner -SaferScanner = Py38SaferScanner if version_info >= (3, 8) else SaferScanner +if version_info >= (3, 11): + SaferScanner = Py311SaferScanner +elif version_info >= (3, 8): + SaferScanner = Py38SaferScanner From d38427a67b1b9684ca85d27a19f321325032dafe Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 8 Dec 2022 10:35:54 -0600 Subject: [PATCH 02/10] add py311 tests to circle Patch by brandonwilliams; reviewed by bereng for CASSANDRA-18094 --- .circleci/config-2_1.yml | 180 ++++- .circleci/config-2_1.yml.mid_res.patch | 112 ++- .circleci/config.yml | 669 ++++++++++++++++ .circleci/config.yml.HIGHRES | 669 ++++++++++++++++ .circleci/config.yml.LOWRES | 669 ++++++++++++++++ .circleci/config.yml.MIDRES | 1016 ++++++++++++++++++++++++ 6 files changed, 3283 insertions(+), 32 deletions(-) diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml index 59581213c0..188984a511 100644 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@ -535,6 +535,10 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - start_j8_cqlsh_tests + - j8_build - j8_cqlsh-dtests-py2-no-vnodes: requires: - start_j8_cqlsh_tests @@ -547,6 +551,10 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311: + requires: + - start_j8_cqlsh_tests + - j8_build # Java 8 cqlsh offheap dtests offheap - start_j8_cqlsh_tests_offheap: type: approval @@ -562,6 +570,10 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j8_cqlsh_tests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_tests_offheap + - j8_build # Java 11 cqlsh dtests - start_j11_cqlsh_tests: type: approval @@ -577,6 +589,10 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j8_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - start_j11_cqlsh_tests @@ -589,6 +605,10 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j8_build # Java 11 cqlsh dtests off-heap - start_j11_cqlsh_tests_offheap: type: approval @@ -604,6 +624,10 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j11_cqlsh_tests_offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh_tests_offheap + - j8_build # Java 8 repeated utest - start_j8_repeated_ant_test: type: approval @@ -835,6 +859,9 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs - j8_cqlsh_dtests_py38_vnode: requires: - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - j8_build - j8_cqlsh-dtests-py2-no-vnodes: requires: - j8_build @@ -844,6 +871,9 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs - j8_cqlsh_dtests_py38: requires: - j8_build + - j8_cqlsh_dtests_py311: + requires: + - j8_build # Java 8 cqlsh dtests offheap - start_j8_cqlsh_dtests_offheap: type: approval @@ -859,6 +889,10 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs requires: - start_j8_cqlsh_dtests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_dtests_offheap + - j8_build # Java 11 cqlsh dtests - j11_cqlsh-dtests-py2-with-vnodes: requires: @@ -869,6 +903,9 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs - j11_cqlsh_dtests_py38_vnode: requires: - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j8_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - j8_build @@ -878,6 +915,9 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs - j11_cqlsh_dtests_py38: requires: - j8_build + - j11_cqlsh_dtests_py311: + requires: + - j8_build # Java 11 cqlsh dtests off-heap - start_j11_cqlsh-dtests-offheap: type: approval @@ -893,6 +933,10 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs requires: - start_j11_cqlsh-dtests-offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j8_build j11_separate_jobs: &j11_separate_jobs jobs: @@ -983,6 +1027,10 @@ j11_separate_jobs: &j11_separate_jobs requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - start_j11_cqlsh_tests @@ -995,6 +1043,10 @@ j11_separate_jobs: &j11_separate_jobs requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -1009,6 +1061,10 @@ j11_separate_jobs: &j11_separate_jobs requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build # specialized unit tests (all run on request) - start_j11_utests_long: type: approval @@ -1151,6 +1207,9 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs - j11_cqlsh_dtests_py38_vnode: requires: - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - j11_build @@ -1160,6 +1219,9 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs - j11_cqlsh_dtests_py38: requires: - j11_build + - j11_cqlsh_dtests_py311: + requires: + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -1174,6 +1236,10 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build # specialized unit tests (all run on request) - start_utests_long: type: approval @@ -1667,6 +1733,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j8_cqlsh_dtests_py311_vnode: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j8_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j8_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j8_cqlsh_dtests_py38_offheap: <<: *j8_par_executor steps: @@ -1685,6 +1769,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j8_cqlsh_dtests_py311_offheap: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j8_dtests_offheap + run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j8_dtests_offheap + pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j8_cqlsh-dtests-py2-no-vnodes: <<: *j8_par_executor steps: @@ -1733,6 +1835,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j8_cqlsh_dtests_py311: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j8_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j8_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j11_cqlsh-dtests-py2-with-vnodes: <<: *j11_par_executor steps: @@ -1811,6 +1931,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j11_cqlsh_dtests_py311_vnode: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j11_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j11_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j11_cqlsh_dtests_py38_offheap: <<: *j11_par_executor steps: @@ -1829,6 +1967,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j11_cqlsh_dtests_py311_offheap: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j11_dtests_offheap + run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j11_dtests_offheap + pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j11_cqlsh-dtests-py2-no-vnodes: <<: *j11_par_executor steps: @@ -1877,6 +2033,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j11_cqlsh_dtests_py311: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j11_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j11_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j8_unit_tests_repeat: <<: *j8_repeated_utest_executor steps: @@ -2379,7 +2553,7 @@ commands: python_version: type: enum default: "3.6" - enum: ["3.6", "3.7", "3.8"] + enum: ["3.6", "3.7", "3.8", "3.11"] steps: - run: name: Configure virtualenv and python Dependencies @@ -2410,7 +2584,7 @@ commands: python_version: type: enum default: "3.6" - enum: ["3.6", "3.7", "3.8"] + enum: ["3.6", "3.7", "3.8", "3.11"] steps: - run: name: Determine Tests to Run (<>) @@ -2452,7 +2626,7 @@ commands: python_version: type: enum default: "3.6" - enum: ["3.6", "3.7", "3.8"] + enum: ["3.6", "3.7", "3.8", "3.11"] steps: - run: name: Run dtests (<>) diff --git a/.circleci/config-2_1.yml.mid_res.patch b/.circleci/config-2_1.yml.mid_res.patch index f3437aeed1..6c3f419e73 100644 --- a/.circleci/config-2_1.yml.mid_res.patch +++ b/.circleci/config-2_1.yml.mid_res.patch @@ -1,5 +1,5 @@ ---- config-2_1.yml 2022-12-23 09:48:14.000000000 -0500 -+++ config-2_1.yml.MIDRES 2022-12-23 09:49:05.000000000 -0500 +--- config-2_1.yml 2023-01-19 06:38:57.093283218 -0600 ++++ config-2_1.yml.MIDRES 2023-01-13 11:55:08.633040364 -0600 @@ -136,14 +136,14 @@ j8_par_executor: &j8_par_executor executor: @@ -37,7 +37,7 @@ +j8_very_large_par_executor: &j8_very_large_par_executor + executor: + name: java8-executor -+ exec_resource_class: large ++ exec_resource_class: xlarge + parallelism: 100 j8_seq_executor: &j8_seq_executor @@ -126,7 +126,7 @@ j8_separate_jobs: &j8_separate_jobs jobs: -@@ -1489,7 +1513,7 @@ +@@ -1555,7 +1579,7 @@ target: testclasslist-system-keyspace-directory j8_dtests_vnode: @@ -135,7 +135,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1503,7 +1527,7 @@ +@@ -1569,7 +1593,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' j8_dtests_offheap: @@ -144,7 +144,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1517,7 +1541,7 @@ +@@ -1583,7 +1607,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' j11_dtests_vnode: @@ -153,7 +153,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1532,7 +1556,7 @@ +@@ -1598,7 +1622,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' j11_dtests_offheap: @@ -162,7 +162,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1547,7 +1571,7 @@ +@@ -1613,7 +1637,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' j8_dtests: @@ -171,7 +171,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1561,7 +1585,7 @@ +@@ -1627,7 +1651,7 @@ pytest_extra_args: '--skip-resource-intensive-tests' j11_dtests: @@ -180,7 +180,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1576,7 +1600,7 @@ +@@ -1642,7 +1666,7 @@ pytest_extra_args: '--skip-resource-intensive-tests' j8_upgrade_dtests: @@ -189,7 +189,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1590,7 +1614,7 @@ +@@ -1656,7 +1680,7 @@ pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' j8_cqlsh-dtests-py2-with-vnodes: @@ -198,7 +198,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1605,7 +1629,7 @@ +@@ -1671,7 +1695,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' j8_cqlsh-dtests-py2-offheap: @@ -207,7 +207,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1620,7 +1644,7 @@ +@@ -1686,7 +1710,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' j8_cqlsh_dtests_py3_vnode: @@ -216,7 +216,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1635,7 +1659,7 @@ +@@ -1701,7 +1725,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j8_cqlsh_dtests_py3_offheap: @@ -225,7 +225,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1650,7 +1674,7 @@ +@@ -1716,7 +1740,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j8_cqlsh_dtests_py38_vnode: @@ -234,25 +234,43 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1668,7 +1692,7 @@ +@@ -1734,7 +1758,7 @@ python_version: '3.8' + j8_cqlsh_dtests_py311_vnode: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -1752,7 +1776,7 @@ + python_version: '3.11' + j8_cqlsh_dtests_py38_offheap: - <<: *j8_par_executor + <<: *j8_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -1686,7 +1710,7 @@ +@@ -1770,7 +1794,7 @@ python_version: '3.8' + j8_cqlsh_dtests_py311_offheap: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -1788,7 +1812,7 @@ + python_version: '3.11' + j8_cqlsh-dtests-py2-no-vnodes: - <<: *j8_par_executor + <<: *j8_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -1701,7 +1725,7 @@ +@@ -1803,7 +1827,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' j8_cqlsh_dtests_py3: @@ -261,7 +279,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1716,7 +1740,7 @@ +@@ -1818,7 +1842,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j8_cqlsh_dtests_py38: @@ -270,16 +288,25 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1734,7 +1758,7 @@ +@@ -1836,7 +1860,7 @@ python_version: '3.8' + j8_cqlsh_dtests_py311: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -1854,7 +1878,7 @@ + python_version: '3.11' + j11_cqlsh-dtests-py2-with-vnodes: - <<: *j11_par_executor + <<: *j11_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -1749,7 +1773,7 @@ +@@ -1869,7 +1893,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' j11_cqlsh-dtests-py2-offheap: @@ -288,7 +315,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1764,7 +1788,7 @@ +@@ -1884,7 +1908,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' j11_cqlsh_dtests_py3_vnode: @@ -297,7 +324,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1779,7 +1803,7 @@ +@@ -1899,7 +1923,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py3_offheap: @@ -306,7 +333,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1794,7 +1818,7 @@ +@@ -1914,7 +1938,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py38_vnode: @@ -315,25 +342,43 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1812,7 +1836,7 @@ +@@ -1932,7 +1956,7 @@ python_version: '3.8' + j11_cqlsh_dtests_py311_vnode: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -1950,7 +1974,7 @@ + python_version: '3.11' + j11_cqlsh_dtests_py38_offheap: - <<: *j11_par_executor + <<: *j11_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -1830,7 +1854,7 @@ +@@ -1968,7 +1992,7 @@ python_version: '3.8' + j11_cqlsh_dtests_py311_offheap: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -1986,7 +2010,7 @@ + python_version: '3.11' + j11_cqlsh-dtests-py2-no-vnodes: - <<: *j11_par_executor + <<: *j11_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -1845,7 +1869,7 @@ +@@ -2001,7 +2025,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' j11_cqlsh_dtests_py3: @@ -342,7 +387,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1860,7 +1884,7 @@ +@@ -2016,7 +2040,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py38: @@ -351,7 +396,16 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2895,3 +2919,4 @@ +@@ -2034,7 +2058,7 @@ + python_version: '3.8' + + j11_cqlsh_dtests_py311: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -3069,3 +3093,4 @@ - store_artifacts: path: ~/cassandra-dtest/logs destination: dtest_logs diff --git a/.circleci/config.yml b/.circleci/config.yml index dae58f89b9..1c25ea0b6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -423,6 +423,106 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -949,6 +1049,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1830,6 +2031,106 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4701,6 +5002,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j11_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6078,6 +6480,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_long_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7664,6 +8167,106 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j8_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -9287,6 +9890,10 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - start_j8_cqlsh_tests + - j8_build - j8_cqlsh-dtests-py2-no-vnodes: requires: - start_j8_cqlsh_tests @@ -9299,6 +9906,10 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311: + requires: + - start_j8_cqlsh_tests + - j8_build - start_j8_cqlsh_tests_offheap: type: approval - j8_cqlsh-dtests-py2-offheap: @@ -9313,6 +9924,10 @@ workflows: requires: - start_j8_cqlsh_tests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_tests_offheap + - j8_build - start_j11_cqlsh_tests: type: approval - j11_cqlsh-dtests-py2-with-vnodes: @@ -9327,6 +9942,10 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j8_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - start_j11_cqlsh_tests @@ -9339,6 +9958,10 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j8_build - start_j11_cqlsh_tests_offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9353,6 +9976,10 @@ workflows: requires: - start_j11_cqlsh_tests_offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh_tests_offheap + - j8_build java8_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9479,6 +10106,9 @@ workflows: - j8_cqlsh_dtests_py38_vnode: requires: - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - j8_build - j8_cqlsh-dtests-py2-no-vnodes: requires: - j8_build @@ -9488,6 +10118,9 @@ workflows: - j8_cqlsh_dtests_py38: requires: - j8_build + - j8_cqlsh_dtests_py311: + requires: + - j8_build - start_j8_cqlsh_dtests_offheap: type: approval - j8_cqlsh-dtests-py2-offheap: @@ -9502,6 +10135,10 @@ workflows: requires: - start_j8_cqlsh_dtests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_dtests_offheap + - j8_build - j11_cqlsh-dtests-py2-with-vnodes: requires: - j8_build @@ -9511,6 +10148,9 @@ workflows: - j11_cqlsh_dtests_py38_vnode: requires: - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j8_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - j8_build @@ -9520,6 +10160,9 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j8_build + - j11_cqlsh_dtests_py311: + requires: + - j8_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9534,6 +10177,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j8_build java11_separate_tests: jobs: - start_j11_build: @@ -9591,6 +10238,10 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - start_j11_cqlsh_tests @@ -9603,6 +10254,10 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9617,6 +10272,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_j11_utests_long: type: approval - j11_utests_long: @@ -9696,6 +10355,9 @@ workflows: - j11_cqlsh_dtests_py38_vnode: requires: - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - j11_build @@ -9705,6 +10367,9 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j11_build + - j11_cqlsh_dtests_py311: + requires: + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9719,6 +10384,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_utests_long: type: approval - j11_utests_long: diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES index 95385e778d..88825e9389 100644 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@ -423,6 +423,106 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -949,6 +1049,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1830,6 +2031,106 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4701,6 +5002,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j11_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6078,6 +6480,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_long_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7664,6 +8167,106 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j8_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -9287,6 +9890,10 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - start_j8_cqlsh_tests + - j8_build - j8_cqlsh-dtests-py2-no-vnodes: requires: - start_j8_cqlsh_tests @@ -9299,6 +9906,10 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311: + requires: + - start_j8_cqlsh_tests + - j8_build - start_j8_cqlsh_tests_offheap: type: approval - j8_cqlsh-dtests-py2-offheap: @@ -9313,6 +9924,10 @@ workflows: requires: - start_j8_cqlsh_tests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_tests_offheap + - j8_build - start_j11_cqlsh_tests: type: approval - j11_cqlsh-dtests-py2-with-vnodes: @@ -9327,6 +9942,10 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j8_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - start_j11_cqlsh_tests @@ -9339,6 +9958,10 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j8_build - start_j11_cqlsh_tests_offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9353,6 +9976,10 @@ workflows: requires: - start_j11_cqlsh_tests_offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh_tests_offheap + - j8_build java8_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9479,6 +10106,9 @@ workflows: - j8_cqlsh_dtests_py38_vnode: requires: - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - j8_build - j8_cqlsh-dtests-py2-no-vnodes: requires: - j8_build @@ -9488,6 +10118,9 @@ workflows: - j8_cqlsh_dtests_py38: requires: - j8_build + - j8_cqlsh_dtests_py311: + requires: + - j8_build - start_j8_cqlsh_dtests_offheap: type: approval - j8_cqlsh-dtests-py2-offheap: @@ -9502,6 +10135,10 @@ workflows: requires: - start_j8_cqlsh_dtests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_dtests_offheap + - j8_build - j11_cqlsh-dtests-py2-with-vnodes: requires: - j8_build @@ -9511,6 +10148,9 @@ workflows: - j11_cqlsh_dtests_py38_vnode: requires: - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j8_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - j8_build @@ -9520,6 +10160,9 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j8_build + - j11_cqlsh_dtests_py311: + requires: + - j8_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9534,6 +10177,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j8_build java11_separate_tests: jobs: - start_j11_build: @@ -9591,6 +10238,10 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - start_j11_cqlsh_tests @@ -9603,6 +10254,10 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9617,6 +10272,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_j11_utests_long: type: approval - j11_utests_long: @@ -9696,6 +10355,9 @@ workflows: - j11_cqlsh_dtests_py38_vnode: requires: - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - j11_build @@ -9705,6 +10367,9 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j11_build + - j11_cqlsh_dtests_py311: + requires: + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9719,6 +10384,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_utests_long: type: approval - j11_utests_long: diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES index dae58f89b9..1c25ea0b6b 100644 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@ -423,6 +423,106 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -949,6 +1049,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1830,6 +2031,106 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4701,6 +5002,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j11_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6078,6 +6480,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_long_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7664,6 +8167,106 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j8_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -9287,6 +9890,10 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - start_j8_cqlsh_tests + - j8_build - j8_cqlsh-dtests-py2-no-vnodes: requires: - start_j8_cqlsh_tests @@ -9299,6 +9906,10 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311: + requires: + - start_j8_cqlsh_tests + - j8_build - start_j8_cqlsh_tests_offheap: type: approval - j8_cqlsh-dtests-py2-offheap: @@ -9313,6 +9924,10 @@ workflows: requires: - start_j8_cqlsh_tests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_tests_offheap + - j8_build - start_j11_cqlsh_tests: type: approval - j11_cqlsh-dtests-py2-with-vnodes: @@ -9327,6 +9942,10 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j8_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - start_j11_cqlsh_tests @@ -9339,6 +9958,10 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j8_build - start_j11_cqlsh_tests_offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9353,6 +9976,10 @@ workflows: requires: - start_j11_cqlsh_tests_offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh_tests_offheap + - j8_build java8_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9479,6 +10106,9 @@ workflows: - j8_cqlsh_dtests_py38_vnode: requires: - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - j8_build - j8_cqlsh-dtests-py2-no-vnodes: requires: - j8_build @@ -9488,6 +10118,9 @@ workflows: - j8_cqlsh_dtests_py38: requires: - j8_build + - j8_cqlsh_dtests_py311: + requires: + - j8_build - start_j8_cqlsh_dtests_offheap: type: approval - j8_cqlsh-dtests-py2-offheap: @@ -9502,6 +10135,10 @@ workflows: requires: - start_j8_cqlsh_dtests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_dtests_offheap + - j8_build - j11_cqlsh-dtests-py2-with-vnodes: requires: - j8_build @@ -9511,6 +10148,9 @@ workflows: - j11_cqlsh_dtests_py38_vnode: requires: - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j8_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - j8_build @@ -9520,6 +10160,9 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j8_build + - j11_cqlsh_dtests_py311: + requires: + - j8_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9534,6 +10177,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j8_build java11_separate_tests: jobs: - start_j11_build: @@ -9591,6 +10238,10 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - start_j11_cqlsh_tests @@ -9603,6 +10254,10 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9617,6 +10272,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_j11_utests_long: type: approval - j11_utests_long: @@ -9696,6 +10355,9 @@ workflows: - j11_cqlsh_dtests_py38_vnode: requires: - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - j11_build @@ -9705,6 +10367,9 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j11_build + - j11_cqlsh_dtests_py311: + requires: + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9719,6 +10384,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_utests_long: type: approval - j11_utests_long: diff --git a/.circleci/config.yml.MIDRES b/.circleci/config.yml.MIDRES index 4519d7da30..430a04f82d 100644 --- a/.circleci/config.yml.MIDRES +++ b/.circleci/config.yml.MIDRES @@ -423,6 +423,106 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -949,6 +1049,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1830,6 +2031,106 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4701,6 +5002,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j11_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6078,6 +6480,107 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_long_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7664,6 +8167,106 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j8_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -9129,12 +9732,24 @@ workflows: requires: - start_j8_unit_tests - j8_build + - start_j8_unit_tests_repeat: + type: approval + - j8_unit_tests_repeat: + requires: + - start_j8_unit_tests_repeat + - j8_build - start_j8_jvm_dtests: type: approval - j8_jvm_dtests: requires: - start_j8_jvm_dtests - j8_build + - start_j8_jvm_dtests_repeat: + type: approval + - j8_jvm_dtests_repeat: + requires: + - start_j8_jvm_dtests_repeat + - j8_build - start_j8_cqlshlib_tests: type: approval - j8_cqlshlib_tests: @@ -9147,6 +9762,12 @@ workflows: requires: - start_j11_unit_tests - j8_build + - start_j11_unit_tests_repeat: + type: approval + - j11_unit_tests_repeat: + requires: + - start_j11_unit_tests_repeat + - j8_build - start_j8_utests_long: type: approval - j8_utests_long: @@ -9159,6 +9780,18 @@ workflows: requires: - start_j11_utests_long - j8_build + - start_j8_utests_long_repeat: + type: approval + - j8_utests_long_repeat: + requires: + - start_j8_utests_long_repeat + - j8_build + - start_j11_utests_long_repeat: + type: approval + - j11_utests_long_repeat: + requires: + - start_j11_utests_long_repeat + - j8_build - start_j8_utests_cdc: type: approval - j8_utests_cdc: @@ -9171,6 +9804,18 @@ workflows: requires: - start_j11_utests_cdc - j8_build + - start_j8_utests_cdc_repeat: + type: approval + - j8_utests_cdc_repeat: + requires: + - start_j8_utests_cdc_repeat + - j8_build + - start_j11_utests_cdc_repeat: + type: approval + - j11_utests_cdc_repeat: + requires: + - start_j11_utests_cdc_repeat + - j8_build - start_j8_utests_compression: type: approval - j8_utests_compression: @@ -9183,6 +9828,18 @@ workflows: requires: - start_j11_utests_compression - j8_build + - start_j8_utests_compression_repeat: + type: approval + - j8_utests_compression_repeat: + requires: + - start_j8_utests_compression_repeat + - j8_build + - start_j11_utests_compression_repeat: + type: approval + - j11_utests_compression_repeat: + requires: + - start_j11_utests_compression_repeat + - j8_build - start_j8_utests_stress: type: approval - j8_utests_stress: @@ -9195,6 +9852,18 @@ workflows: requires: - start_j11_utests_stress - j8_build + - start_j8_utests_stress_repeat: + type: approval + - j8_utests_stress_repeat: + requires: + - start_j8_utests_stress_repeat + - j8_build + - start_j11_utests_stress_repeat: + type: approval + - j11_utests_stress_repeat: + requires: + - start_j11_utests_stress_repeat + - j8_build - start_j8_utests_fqltool: type: approval - j8_utests_fqltool: @@ -9207,6 +9876,18 @@ workflows: requires: - start_j11_utests_fqltool - j8_build + - start_j8_utests_fqltool_repeat: + type: approval + - j8_utests_fqltool_repeat: + requires: + - start_j8_utests_fqltool_repeat + - j8_build + - start_j11_utests_fqltool_repeat: + type: approval + - j11_utests_fqltool_repeat: + requires: + - start_j11_utests_fqltool_repeat + - j8_build - start_j8_utests_system_keyspace_directory: type: approval - j8_utests_system_keyspace_directory: @@ -9219,6 +9900,18 @@ workflows: requires: - start_j11_utests_system_keyspace_directory - j8_build + - start_j8_utests_system_keyspace_directory_repeat: + type: approval + - j8_utests_system_keyspace_directory_repeat: + requires: + - start_j8_utests_system_keyspace_directory_repeat + - j8_build + - start_j11_utests_system_keyspace_directory_repeat: + type: approval + - j11_utests_system_keyspace_directory_repeat: + requires: + - start_j11_utests_system_keyspace_directory_repeat + - j8_build - start_j8_dtest_jars_build: type: approval - j8_dtest_jars_build: @@ -9231,48 +9924,96 @@ workflows: requires: - start_jvm_upgrade_dtests - j8_dtest_jars_build + - start_jvm_upgrade_dtests_repeat: + type: approval + - j8_jvm_upgrade_dtests_repeat: + requires: + - start_jvm_upgrade_dtests_repeat + - j8_dtest_jars_build - start_j8_dtests: type: approval - j8_dtests: requires: - start_j8_dtests - j8_build + - start_j8_dtests_repeat: + type: approval + - j8_dtests_repeat: + requires: + - start_j8_dtests_repeat + - j8_build - start_j8_dtests_vnode: type: approval - j8_dtests_vnode: requires: - start_j8_dtests_vnode - j8_build + - start_j8_dtests_vnode_repeat: + type: approval + - j8_dtests_vnode_repeat: + requires: + - start_j8_dtests_vnode_repeat + - j8_build - start_j8_dtests_offheap: type: approval - j8_dtests_offheap: requires: - start_j8_dtests_offheap - j8_build + - start_j8_dtests_offheap_repeat: + type: approval + - j8_dtests_offheap_repeat: + requires: + - start_j8_dtests_offheap_repeat + - j8_build - start_j11_dtests: type: approval - j11_dtests: requires: - start_j11_dtests - j8_build + - start_j11_dtests_repeat: + type: approval + - j11_dtests_repeat: + requires: + - start_j11_dtests_repeat + - j8_build - start_j11_dtests_vnode: type: approval - j11_dtests_vnode: requires: - start_j11_dtests_vnode - j8_build + - start_j11_dtests_vnode_repeat: + type: approval + - j11_dtests_vnode_repeat: + requires: + - start_j11_dtests_vnode_repeat + - j8_build - start_j11_dtests_offheap: type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j8_build + - start_j11_dtests_offheap_repeat: + type: approval + - j11_dtests_offheap_repeat: + requires: + - start_j11_dtests_offheap_repeat + - j8_build - start_upgrade_tests: type: approval - j8_upgrade_dtests: requires: - start_upgrade_tests - j8_build + - start_j8_upgrade_dtests_repeat: + type: approval + - j8_upgrade_dtests_repeat: + requires: + - start_j8_upgrade_dtests_repeat + - j8_build - start_j8_cqlsh_tests: type: approval - j8_cqlsh-dtests-py2-with-vnodes: @@ -9287,6 +10028,10 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - start_j8_cqlsh_tests + - j8_build - j8_cqlsh-dtests-py2-no-vnodes: requires: - start_j8_cqlsh_tests @@ -9299,6 +10044,10 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311: + requires: + - start_j8_cqlsh_tests + - j8_build - start_j8_cqlsh_tests_offheap: type: approval - j8_cqlsh-dtests-py2-offheap: @@ -9313,6 +10062,10 @@ workflows: requires: - start_j8_cqlsh_tests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_tests_offheap + - j8_build - start_j11_cqlsh_tests: type: approval - j11_cqlsh-dtests-py2-with-vnodes: @@ -9327,6 +10080,10 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j8_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - start_j11_cqlsh_tests @@ -9339,6 +10096,10 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j8_build - start_j11_cqlsh_tests_offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9353,6 +10114,22 @@ workflows: requires: - start_j11_cqlsh_tests_offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh_tests_offheap + - j8_build + - start_j8_repeated_ant_test: + type: approval + - j8_repeated_ant_test: + requires: + - start_j8_repeated_ant_test + - j8_build + - start_j11_repeated_ant_test: + type: approval + - j11_repeated_ant_test: + requires: + - start_j11_repeated_ant_test + - j8_build java8_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9363,15 +10140,24 @@ workflows: - j8_unit_tests: requires: - j8_build + - j8_unit_tests_repeat: + requires: + - j8_build - j8_jvm_dtests: requires: - j8_build + - j8_jvm_dtests_repeat: + requires: + - j8_build - j8_cqlshlib_tests: requires: - j8_build - j11_unit_tests: requires: - j8_build + - j11_unit_tests_repeat: + requires: + - j8_build - start_utests_long: type: approval - j8_utests_long: @@ -9382,6 +10168,14 @@ workflows: requires: - start_utests_long - j8_build + - j8_utests_long_repeat: + requires: + - start_utests_long + - j8_build + - j11_utests_long_repeat: + requires: + - start_utests_long + - j8_build - start_utests_cdc: type: approval - j8_utests_cdc: @@ -9392,6 +10186,14 @@ workflows: requires: - start_utests_cdc - j8_build + - j8_utests_cdc_repeat: + requires: + - start_utests_cdc + - j8_build + - j11_utests_cdc_repeat: + requires: + - start_utests_cdc + - j8_build - start_utests_compression: type: approval - j8_utests_compression: @@ -9402,6 +10204,14 @@ workflows: requires: - start_utests_compression - j8_build + - j8_utests_compression_repeat: + requires: + - start_utests_compression + - j8_build + - j11_utests_compression_repeat: + requires: + - start_utests_compression + - j8_build - start_utests_stress: type: approval - j8_utests_stress: @@ -9412,6 +10222,14 @@ workflows: requires: - start_utests_stress - j8_build + - j8_utests_stress_repeat: + requires: + - start_utests_stress + - j8_build + - j11_utests_stress_repeat: + requires: + - start_utests_stress + - j8_build - start_utests_fqltool: type: approval - j8_utests_fqltool: @@ -9422,6 +10240,14 @@ workflows: requires: - start_utests_fqltool - j8_build + - j8_utests_fqltool_repeat: + requires: + - start_utests_fqltool + - j8_build + - j11_utests_fqltool_repeat: + requires: + - start_utests_fqltool + - j8_build - start_utests_system_keyspace_directory: type: approval - j8_utests_system_keyspace_directory: @@ -9431,6 +10257,13 @@ workflows: requires: - start_utests_system_keyspace_directory - j8_build + - j8_utests_system_keyspace_directory_repeat: + requires: + - j8_build + - j11_utests_system_keyspace_directory_repeat: + requires: + - start_utests_system_keyspace_directory + - j8_build - start_jvm_upgrade_dtests: type: approval - j8_dtest_jars_build: @@ -9440,36 +10273,63 @@ workflows: - j8_jvm_upgrade_dtests: requires: - j8_dtest_jars_build + - j8_jvm_upgrade_dtests_repeat: + requires: + - j8_dtest_jars_build - j8_dtests: requires: - j8_build + - j8_dtests_repeat: + requires: + - j8_build - j8_dtests_vnode: requires: - j8_build + - j8_dtests_vnode_repeat: + requires: + - j8_build - start_j8_dtests_offheap: type: approval - j8_dtests_offheap: requires: - start_j8_dtests_offheap - j8_build + - j8_dtests_offheap_repeat: + requires: + - start_j8_dtests_offheap + - j8_build - j11_dtests: requires: - j8_build + - j11_dtests_repeat: + requires: + - j8_build - j11_dtests_vnode: requires: - j8_build + - j11_dtests_vnode_repeat: + requires: + - j8_build - start_j11_dtests_offheap: type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j8_build + - j11_dtests_offheap_repeat: + requires: + - start_j11_dtests_offheap + - j8_build - start_upgrade_tests: type: approval - j8_upgrade_dtests: requires: - j8_build - start_upgrade_tests + - j8_upgrade_dtests_repeat: + requires: + - j8_build + - start_upgrade_tests - j8_cqlsh-dtests-py2-with-vnodes: requires: - j8_build @@ -9479,6 +10339,9 @@ workflows: - j8_cqlsh_dtests_py38_vnode: requires: - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - j8_build - j8_cqlsh-dtests-py2-no-vnodes: requires: - j8_build @@ -9488,6 +10351,9 @@ workflows: - j8_cqlsh_dtests_py38: requires: - j8_build + - j8_cqlsh_dtests_py311: + requires: + - j8_build - start_j8_cqlsh_dtests_offheap: type: approval - j8_cqlsh-dtests-py2-offheap: @@ -9502,6 +10368,10 @@ workflows: requires: - start_j8_cqlsh_dtests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_dtests_offheap + - j8_build - j11_cqlsh-dtests-py2-with-vnodes: requires: - j8_build @@ -9511,6 +10381,9 @@ workflows: - j11_cqlsh_dtests_py38_vnode: requires: - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j8_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - j8_build @@ -9520,6 +10393,9 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j8_build + - j11_cqlsh_dtests_py311: + requires: + - j8_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9534,6 +10410,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j8_build java11_separate_tests: jobs: - start_j11_build: @@ -9547,12 +10427,24 @@ workflows: requires: - start_j11_unit_tests - j11_build + - start_j11_unit_tests_repeat: + type: approval + - j11_unit_tests_repeat: + requires: + - start_j11_unit_tests_repeat + - j11_build - start_j11_jvm_dtests: type: approval - j11_jvm_dtests: requires: - start_j11_jvm_dtests - j11_build + - start_j11_jvm_dtests_repeat: + type: approval + - j11_jvm_dtests_repeat: + requires: + - start_j11_jvm_dtests_repeat + - j11_build - start_j11_cqlshlib_tests: type: approval - j11_cqlshlib_tests: @@ -9571,12 +10463,30 @@ workflows: requires: - start_j11_dtests_vnode - j11_build + - start_j11_dtests_repeat: + type: approval + - j11_dtests_repeat: + requires: + - start_j11_dtests_repeat + - j11_build + - start_j11_dtests_vnode_repeat: + type: approval + - j11_dtests_vnode_repeat: + requires: + - start_j11_dtests_vnode_repeat + - j11_build - start_j11_dtests_offheap: type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j11_build + - start_j11_dtests_offheap_repeat: + type: approval + - j11_dtests_offheap_repeat: + requires: + - start_j11_dtests_offheap_repeat + - j11_build - start_j11_cqlsh_tests: type: approval - j11_cqlsh-dtests-py2-with-vnodes: @@ -9591,6 +10501,10 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - start_j11_cqlsh_tests @@ -9603,6 +10517,10 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9617,42 +10535,88 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_j11_utests_long: type: approval - j11_utests_long: requires: - start_j11_utests_long - j11_build + - start_j11_utests_long_repeat: + type: approval + - j11_utests_long_repeat: + requires: + - start_j11_utests_long_repeat + - j11_build - start_j11_utests_cdc: type: approval - j11_utests_cdc: requires: - start_j11_utests_cdc - j11_build + - start_j11_utests_cdc_repeat: + type: approval + - j11_utests_cdc_repeat: + requires: + - start_j11_utests_cdc_repeat + - j11_build - start_j11_utests_compression: type: approval - j11_utests_compression: requires: - start_j11_utests_compression - j11_build + - start_j11_utests_compression_repeat: + type: approval + - j11_utests_compression_repeat: + requires: + - start_j11_utests_compression_repeat + - j11_build - start_j11_utests_stress: type: approval - j11_utests_stress: requires: - start_j11_utests_stress - j11_build + - start_j11_utests_stress_repeat: + type: approval + - j11_utests_stress_repeat: + requires: + - start_j11_utests_stress_repeat + - j11_build - start_j11_utests_fqltool: type: approval - j11_utests_fqltool: requires: - start_j11_utests_fqltool - j11_build + - start_j11_utests_fqltool_repeat: + type: approval + - j11_utests_fqltool_repeat: + requires: + - start_j11_utests_fqltool_repeat + - j11_build - start_j11_utests_system_keyspace_directory: type: approval - j11_utests_system_keyspace_directory: requires: - start_j11_utests_system_keyspace_directory - j11_build + - start_j11_utests_system_keyspace_directory_repeat: + type: approval + - j11_utests_system_keyspace_directory_repeat: + requires: + - start_j11_utests_system_keyspace_directory_repeat + - j11_build + - start_j11_repeated_ant_test: + type: approval + - j11_repeated_ant_test: + requires: + - start_j11_repeated_ant_test + - j11_build java11_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9663,9 +10627,15 @@ workflows: - j11_unit_tests: requires: - j11_build + - j11_unit_tests_repeat: + requires: + - j11_build - j11_jvm_dtests: requires: - j11_build + - j11_jvm_dtests_repeat: + requires: + - j11_build - j11_cqlshlib_tests: requires: - j11_build @@ -9678,15 +10648,27 @@ workflows: - j11_dtests: requires: - j11_build + - j11_dtests_repeat: + requires: + - j11_build - j11_dtests_vnode: requires: - j11_build + - j11_dtests_vnode_repeat: + requires: + - j11_build - start_j11_dtests_offheap: type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j11_build + - start_j11_dtests_offheap_repeat: + type: approval + - j11_dtests_offheap_repeat: + requires: + - start_j11_dtests_offheap_repeat + - j11_build - j11_cqlsh-dtests-py2-with-vnodes: requires: - j11_build @@ -9696,6 +10678,9 @@ workflows: - j11_cqlsh_dtests_py38_vnode: requires: - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j11_build - j11_cqlsh-dtests-py2-no-vnodes: requires: - j11_build @@ -9705,6 +10690,9 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j11_build + - j11_cqlsh_dtests_py311: + requires: + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh-dtests-py2-offheap: @@ -9719,39 +10707,67 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_utests_long: type: approval - j11_utests_long: requires: - start_utests_long - j11_build + - j11_utests_long_repeat: + requires: + - start_utests_long + - j11_build - start_utests_cdc: type: approval - j11_utests_cdc: requires: - start_utests_cdc - j11_build + - j11_utests_cdc_repeat: + requires: + - start_utests_cdc + - j11_build - start_utests_compression: type: approval - j11_utests_compression: requires: - start_utests_compression - j11_build + - j11_utests_compression_repeat: + requires: + - start_utests_compression + - j11_build - start_utests_stress: type: approval - j11_utests_stress: requires: - start_utests_stress - j11_build + - j11_utests_stress_repeat: + requires: + - start_utests_stress + - j11_build - start_utests_fqltool: type: approval - j11_utests_fqltool: requires: - start_utests_fqltool - j11_build + - j11_utests_fqltool_repeat: + requires: + - start_utests_fqltool + - j11_build - start_utests_system_keyspace_directory: type: approval - j11_utests_system_keyspace_directory: requires: - start_utests_system_keyspace_directory - j11_build + - j11_utests_system_keyspace_directory_repeat: + requires: + - start_utests_system_keyspace_directory + - j11_build From 563a26c6b1493f46ebb597085ae589f1835acd50 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Fri, 16 Dec 2022 10:09:39 -0600 Subject: [PATCH 03/10] get newest pip in cassandra-cqlsh-tests.sh Patch by brandonwilliams; reviewed by bereng for CASSANDRA-18121 --- pylib/cassandra-cqlsh-tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh index 35d759bf8a..2b14941856 100755 --- a/pylib/cassandra-cqlsh-tests.sh +++ b/pylib/cassandra-cqlsh-tests.sh @@ -81,6 +81,8 @@ fi set -e # enable immediate exit if venv setup fails virtualenv --python=$PYTHON_VERSION venv source venv/bin/activate +# 3.11 needs the newest pip +curl -sS https://bootstrap.pypa.io/get-pip.py | $PYTHON_VERSION pip install -r ${CASSANDRA_DIR}/pylib/requirements.txt pip freeze From f6933a041901861e526b44af6fc974b29b33982e Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Fri, 16 Dec 2022 12:29:57 -0600 Subject: [PATCH 04/10] convert cqlshlib from nose to pytest Patch by brandonwilliams; reviewed by bereng for CASSANDRA-18088 --- pylib/cassandra-cqlsh-tests.sh | 2 +- pylib/cqlshlib/test/cassconnect.py | 6 +++--- pylib/cqlshlib/test/test_sslhandling.py | 8 ++++---- pylib/pytest.ini | 2 ++ pylib/requirements.txt | 3 +-- 5 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 pylib/pytest.ini diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh index 2b14941856..f039ba8cf8 100755 --- a/pylib/cassandra-cqlsh-tests.sh +++ b/pylib/cassandra-cqlsh-tests.sh @@ -129,7 +129,7 @@ ccm start --wait-for-binary-proto cd ${CASSANDRA_DIR}/pylib/cqlshlib/ set +e # disable immediate exit from this point -nosetests +pytest RETURN="$?" ccm remove diff --git a/pylib/cqlshlib/test/cassconnect.py b/pylib/cqlshlib/test/cassconnect.py index c4eae0ec8a..909e88ae62 100644 --- a/pylib/cqlshlib/test/cassconnect.py +++ b/pylib/cqlshlib/test/cassconnect.py @@ -20,7 +20,7 @@ import io import os.path import random import string -from nose.tools import nottest +import pytest from .basecase import TEST_HOST, TEST_PORT, cql, cqlsh, cqlshlog, policy, quote_name, test_dir from .run_cqlsh import run_cqlsh, call_cqlsh @@ -152,7 +152,7 @@ def cql_rule_set(): class DEFAULTVAL: pass -@nottest +@pytest.mark.skip(reason="not a test") def testrun_cqlsh(keyspace=DEFAULTVAL, **kwargs): # use a positive default sentinel so that keyspace=None can be used # to override the default behavior @@ -161,7 +161,7 @@ def testrun_cqlsh(keyspace=DEFAULTVAL, **kwargs): return run_cqlsh(keyspace=keyspace, **kwargs) -@nottest +@pytest.mark.skip(reason="not a test") def testcall_cqlsh(keyspace=None, **kwargs): if keyspace is None: keyspace = get_keyspace() diff --git a/pylib/cqlshlib/test/test_sslhandling.py b/pylib/cqlshlib/test/test_sslhandling.py index 347fe2a114..a96089d866 100644 --- a/pylib/cqlshlib/test/test_sslhandling.py +++ b/pylib/cqlshlib/test/test_sslhandling.py @@ -17,11 +17,11 @@ from cassandra.policies import SimpleConvictionPolicy from cassandra.pool import Host from cqlshlib.sslhandling import ssl_settings -from nose.tools import assert_raises import unittest import os import ssl +import pytest class SslSettingsTest(unittest.TestCase): @@ -53,7 +53,7 @@ class SslSettingsTest(unittest.TestCase): def test_invalid_ssl_versions_from_env(self): msg = "invalid_ssl is not a valid SSL protocol, please use one of TLSv1, TLSv1_1, or TLSv1_2" - with assert_raises(SystemExit) as error: + with pytest.raises(SystemExit) as error: self._test_ssl_version_from_env('invalid_ssl') assert msg == error.exception.message @@ -69,7 +69,7 @@ class SslSettingsTest(unittest.TestCase): def test_invalid_ssl_version_config(self): msg = "invalid_ssl is not a valid SSL protocol, please use one of TLSv1, TLSv1_1, or TLSv1_2" - with assert_raises(SystemExit) as error: + with pytest.raises(SystemExit) as error: ssl_settings(self.host, os.path.join('test', 'config', 'sslhandling_invalid.config')) assert msg in error.exception.message - \ No newline at end of file + diff --git a/pylib/pytest.ini b/pylib/pytest.ini new file mode 100644 index 0000000000..d360f07d68 --- /dev/null +++ b/pylib/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --junit-xml=nosetests.xml diff --git a/pylib/requirements.txt b/pylib/requirements.txt index 59ff479456..8788690638 100644 --- a/pylib/requirements.txt +++ b/pylib/requirements.txt @@ -11,8 +11,7 @@ docopt enum34 flaky mock -nose -nose-test-select +pytest parse pycodestyle psutil From b43293e200dd95c98bfb3ee436fe05bb18da114f Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Tue, 17 Jan 2023 14:18:56 -0600 Subject: [PATCH 05/10] Upgrade cython Patch by brandonwilliams; reviewed by bereng for CASSANDRA-18121 --- pylib/cassandra-cqlsh-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh index f039ba8cf8..b2d2c500ae 100755 --- a/pylib/cassandra-cqlsh-tests.sh +++ b/pylib/cassandra-cqlsh-tests.sh @@ -89,7 +89,7 @@ pip freeze if [ "$cython" = "yes" ]; then TESTSUITE_NAME="${TESTSUITE_NAME}.cython" - pip install "Cython>=0.27.2,<0.28" + pip install "Cython>=0.29.15,<3.0" cd pylib/; python setup.py build_ext --inplace cd ${WORKSPACE} else From cd38edbb41bdf0babdb77c62f09650ae0a28f77c Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Sun, 4 Dec 2022 13:05:19 -0600 Subject: [PATCH 06/10] Accommodate python 3.11 Patch by brandonwilliams; reviewed by bereng for CASSANDRA-18088 --- CHANGES.txt | 1 + pylib/cqlshlib/saferscanner.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1352d00b26..ed0e5dc731 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,7 @@ * Streaming progress virtual table lock contention can trigger TCP_USER_TIMEOUT and fail streaming (CASSANDRA-18110) * Fix perpetual load of denylist on read in cases where denylist can never be loaded (CASSANDRA-18116) Merged from 4.0: + * Add support for python 3.11 (CASSANDRA-18088) * Fix formatting of duration in cqlsh (CASSANDRA-18141) * Fix sstable loading of keyspaces named snapshots or backups (CASSANDRA-14013) * Avoid ConcurrentModificationException in STCS/DTCS/TWCS.getSSTables (CASSANDRA-17977) diff --git a/pylib/cqlshlib/saferscanner.py b/pylib/cqlshlib/saferscanner.py index 3cc343033a..5afd8ef172 100644 --- a/pylib/cqlshlib/saferscanner.py +++ b/pylib/cqlshlib/saferscanner.py @@ -19,7 +19,11 @@ # regex in-pattern flags. Any of those can break correct operation of Scanner. import re -from sre_constants import BRANCH, SUBPATTERN, GROUPREF, GROUPREF_IGNORE, GROUPREF_EXISTS +import six +try: + from sre_constants import BRANCH, SUBPATTERN, GROUPREF, GROUPREF_IGNORE, GROUPREF_EXISTS +except ImportError: + from re._constants import BRANCH, SUBPATTERN, GROUPREF, GROUPREF_IGNORE, GROUPREF_EXISTS from sys import version_info @@ -81,4 +85,24 @@ class Py38SaferScanner(SaferScannerBase): self.scanner = re.sre_compile.compile(p) -SaferScanner = Py38SaferScanner if version_info >= (3, 8) else Py36SaferScanner +class Py311SaferScanner(SaferScannerBase): + + def __init__(self, lexicon, flags=0): + self.lexicon = lexicon + p = [] + s = re._parser.State() + s.flags = flags + for phrase, action in lexicon: + gid = s.opengroup() + p.append(re._parser.SubPattern(s, [(SUBPATTERN, (gid, 0, 0, re._parser.parse(phrase, flags))), ])) + s.closegroup(gid, p[-1]) + p = re._parser.SubPattern(s, [(BRANCH, (None, p))]) + self.p = p + self.scanner = re._compiler.compile(p) + + +SaferScanner = Py36SaferScanner if six.PY3 else Py2SaferScanner +if version_info >= (3, 11): + SaferScanner = Py311SaferScanner +elif version_info >= (3, 8): + SaferScanner = Py38SaferScanner From f66e7dcf33309b0b910f4d5199b00181c26436a4 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 8 Dec 2022 11:47:03 -0600 Subject: [PATCH 07/10] add py311 tests to circle Patch by brandonwilliams; reviewed by bereng for CASSANDRA-18094 --- .circleci/config-2_1.yml | 180 +++- .circleci/config-2_1.yml.high_res.patch | 14 +- .circleci/config-2_1.yml.mid_res.patch | 106 +- .circleci/config.yml | 729 ++++++++++++++ .circleci/config.yml.HIGHRES | 729 ++++++++++++++ .circleci/config.yml.LOWRES | 729 ++++++++++++++ .circleci/config.yml.MIDRES | 1193 +++++++++++++++++++++++ 7 files changed, 3644 insertions(+), 36 deletions(-) diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml index 3f83b74094..b3341d0f17 100644 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@ -647,10 +647,18 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311: + requires: + - start_j8_cqlsh_tests + - j8_build - j8_cqlsh_dtests_py38_vnode: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - start_j8_cqlsh_tests + - j8_build # Java 8 cqlsh offheap dtests offheap - start_j8_cqlsh_tests_offheap: type: approval @@ -662,6 +670,10 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j8_cqlsh_tests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_tests_offheap + - j8_build # Java 11 cqlsh dtests - start_j11_cqlsh_tests: type: approval @@ -677,10 +689,18 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j8_build - j11_cqlsh_dtests_py38_vnode: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j8_build # Java 11 cqlsh dtests off-heap - start_j11_cqlsh_tests_offheap: type: approval @@ -692,6 +712,10 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j11_cqlsh_tests_offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh_tests_offheap + - j8_build # Java 8 repeated utest - start_j8_repeated_ant_test: type: approval @@ -1038,9 +1062,15 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs - j8_cqlsh_dtests_py38: requires: - j8_build + - j8_cqlsh_dtests_py311: + requires: + - j8_build - j8_cqlsh_dtests_py38_vnode: requires: - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - j8_build # Java 8 cqlsh dtests offheap - start_j8_cqlsh_dtests_offheap: type: approval @@ -1052,6 +1082,10 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs requires: - start_j8_cqlsh_dtests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_dtests_offheap + - j8_build # Java 11 cqlsh dtests - j11_cqlsh_dtests_py3: requires: @@ -1062,9 +1096,15 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs - j11_cqlsh_dtests_py38: requires: - j8_build + - j11_cqlsh_dtests_py311: + requires: + - j8_build - j11_cqlsh_dtests_py38_vnode: requires: - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j8_build # Java 11 cqlsh dtests off-heap - start_j11_cqlsh-dtests-offheap: type: approval @@ -1076,6 +1116,10 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs requires: - start_j11_cqlsh-dtests-offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j8_build j11_separate_jobs: &j11_separate_jobs jobs: @@ -1196,10 +1240,18 @@ j11_separate_jobs: &j11_separate_jobs requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j11_build - j11_cqlsh_dtests_py38_vnode: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -1210,6 +1262,10 @@ j11_separate_jobs: &j11_separate_jobs requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build # specialized unit tests (all run on request) - start_j11_utests_long: type: approval @@ -1392,9 +1448,15 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs - j11_cqlsh_dtests_py38: requires: - j11_build + - j11_cqlsh_dtests_py311: + requires: + - j11_build - j11_cqlsh_dtests_py38_vnode: requires: - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -1405,6 +1467,10 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build # specialized unit tests (all run on request) - start_utests_long: type: approval @@ -1919,6 +1985,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j8_cqlsh_dtests_py311_vnode: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j8_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j8_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j8_cqlsh_dtests_py38_offheap: <<: *j8_par_executor steps: @@ -1937,6 +2021,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j8_cqlsh_dtests_py311_offheap: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j8_dtests_offheap + run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j8_dtests_offheap + pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j8_cqlsh_dtests_py3: <<: *j8_par_executor steps: @@ -1970,6 +2072,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j8_cqlsh_dtests_py311: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j8_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j8_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j11_cqlsh_dtests_py3_vnode: <<: *j11_par_executor steps: @@ -2018,6 +2138,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j11_cqlsh_dtests_py311_vnode: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j11_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j11_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j11_cqlsh_dtests_py38_offheap: <<: *j11_par_executor steps: @@ -2036,6 +2174,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j11_cqlsh_dtests_py311_offheap: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j11_dtests_offheap + run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j11_dtests_offheap + pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j11_cqlsh_dtests_py3: <<: *j11_par_executor steps: @@ -2069,6 +2225,24 @@ jobs: extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.8' python_version: '3.8' + j11_cqlsh_dtests_py311: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv: + python_version: '3.11' + - create_dtest_containers: + file_tag: j11_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + python_version: '3.11' + - run_dtests: + file_tag: j11_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.11' + python_version: '3.11' + j8_dtests_large_vnode: <<: *j8_par_executor steps: @@ -2761,7 +2935,7 @@ commands: python_version: type: enum default: "3.6" - enum: ["3.6", "3.7", "3.8"] + enum: ["3.6", "3.7", "3.8", "3.11"] steps: - run: name: Configure virtualenv and python Dependencies @@ -2792,7 +2966,7 @@ commands: python_version: type: enum default: "3.6" - enum: ["3.6", "3.7", "3.8"] + enum: ["3.6", "3.7", "3.8", "3.11"] steps: - run: name: Determine Tests to Run (<>) @@ -2834,7 +3008,7 @@ commands: python_version: type: enum default: "3.6" - enum: ["3.6", "3.7", "3.8"] + enum: ["3.6", "3.7", "3.8", "3.11"] steps: - run: name: Run dtests (<>) diff --git a/.circleci/config-2_1.yml.high_res.patch b/.circleci/config-2_1.yml.high_res.patch index ded21fff8c..d346a588e7 100644 --- a/.circleci/config-2_1.yml.high_res.patch +++ b/.circleci/config-2_1.yml.high_res.patch @@ -1,5 +1,5 @@ ---- config-2_1.yml 2022-11-15 12:30:50.000000000 -0500 -+++ config-2_1.yml.HIGHRES 2022-11-15 12:35:46.000000000 -0500 +--- config-2_1.yml 2022-12-08 11:47:59.977505612 -0600 ++++ config-2_1.yml.HIGHRES 2022-12-08 11:56:04.308987398 -0600 @@ -157,14 +157,14 @@ j8_par_executor: &j8_par_executor executor: @@ -117,8 +117,8 @@ j8_separate_jobs: &j8_separate_jobs jobs: -@@ -2070,7 +2076,7 @@ - python_version: '3.8' +@@ -2244,7 +2250,7 @@ + python_version: '3.11' j8_dtests_large_vnode: - <<: *j8_par_executor @@ -126,7 +126,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2084,7 +2090,7 @@ +@@ -2258,7 +2264,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests' j8_dtests_large: @@ -135,7 +135,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2098,7 +2104,7 @@ +@@ -2272,7 +2278,7 @@ pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests' j11_dtests_large_vnode: @@ -144,7 +144,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2112,7 +2118,7 @@ +@@ -2286,7 +2292,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests' j11_dtests_large: diff --git a/.circleci/config-2_1.yml.mid_res.patch b/.circleci/config-2_1.yml.mid_res.patch index db78846f2e..3d2ea42db8 100644 --- a/.circleci/config-2_1.yml.mid_res.patch +++ b/.circleci/config-2_1.yml.mid_res.patch @@ -1,5 +1,5 @@ ---- config-2_1.yml 2022-12-23 10:10:35.000000000 -0500 -+++ config-2_1.yml.MIDRES 2022-12-23 10:11:05.000000000 -0500 +--- config-2_1.yml 2023-01-19 06:40:13.405500420 -0600 ++++ config-2_1.yml.MIDRES 2023-01-13 11:58:24.226316122 -0600 @@ -157,14 +157,14 @@ j8_par_executor: &j8_par_executor executor: @@ -32,7 +32,7 @@ +j8_very_large_par_executor: &j8_very_large_par_executor + executor: + name: java8-executor -+ exec_resource_class: large ++ exec_resource_class: xlarge + parallelism: 100 + j8_medium_par_executor: &j8_medium_par_executor @@ -134,7 +134,7 @@ j8_separate_jobs: &j8_separate_jobs jobs: -@@ -1771,7 +1795,7 @@ +@@ -1837,7 +1861,7 @@ target: testclasslist-system-keyspace-directory j8_dtests_vnode: @@ -143,7 +143,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1785,7 +1809,7 @@ +@@ -1851,7 +1875,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' j8_dtests_offheap: @@ -152,7 +152,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1799,7 +1823,7 @@ +@@ -1865,7 +1889,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' j11_dtests_vnode: @@ -161,7 +161,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1814,7 +1838,7 @@ +@@ -1880,7 +1904,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' j11_dtests_offheap: @@ -170,7 +170,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1829,7 +1853,7 @@ +@@ -1895,7 +1919,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' j8_dtests: @@ -179,7 +179,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1843,7 +1867,7 @@ +@@ -1909,7 +1933,7 @@ pytest_extra_args: '--skip-resource-intensive-tests' j11_dtests: @@ -188,7 +188,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1858,7 +1882,7 @@ +@@ -1924,7 +1948,7 @@ pytest_extra_args: '--skip-resource-intensive-tests' j8_upgrade_dtests: @@ -197,7 +197,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1872,7 +1896,7 @@ +@@ -1938,7 +1962,7 @@ pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' j8_cqlsh_dtests_py3_vnode: @@ -206,7 +206,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1887,7 +1911,7 @@ +@@ -1953,7 +1977,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j8_cqlsh_dtests_py3_offheap: @@ -215,7 +215,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1902,7 +1926,7 @@ +@@ -1968,7 +1992,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j8_cqlsh_dtests_py38_vnode: @@ -224,25 +224,43 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1920,7 +1944,7 @@ +@@ -1986,7 +2010,7 @@ python_version: '3.8' + j8_cqlsh_dtests_py311_vnode: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -2004,7 +2028,7 @@ + python_version: '3.11' + j8_cqlsh_dtests_py38_offheap: - <<: *j8_par_executor + <<: *j8_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -1938,7 +1962,7 @@ +@@ -2022,7 +2046,7 @@ python_version: '3.8' + j8_cqlsh_dtests_py311_offheap: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -2040,7 +2064,7 @@ + python_version: '3.11' + j8_cqlsh_dtests_py3: - <<: *j8_par_executor + <<: *j8_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -1953,7 +1977,7 @@ +@@ -2055,7 +2079,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j8_cqlsh_dtests_py38: @@ -251,16 +269,25 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1971,7 +1995,7 @@ +@@ -2073,7 +2097,7 @@ python_version: '3.8' + j8_cqlsh_dtests_py311: +- <<: *j8_par_executor ++ <<: *j8_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -2091,7 +2115,7 @@ + python_version: '3.11' + j11_cqlsh_dtests_py3_vnode: - <<: *j11_par_executor + <<: *j11_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -1986,7 +2010,7 @@ +@@ -2106,7 +2130,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py3_offheap: @@ -269,7 +296,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2001,7 +2025,7 @@ +@@ -2121,7 +2145,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py38_vnode: @@ -278,25 +305,43 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2019,7 +2043,7 @@ +@@ -2139,7 +2163,7 @@ python_version: '3.8' + j11_cqlsh_dtests_py311_vnode: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -2157,7 +2181,7 @@ + python_version: '3.11' + j11_cqlsh_dtests_py38_offheap: - <<: *j11_par_executor + <<: *j11_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -2037,7 +2061,7 @@ +@@ -2175,7 +2199,7 @@ python_version: '3.8' + j11_cqlsh_dtests_py311_offheap: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -2193,7 +2217,7 @@ + python_version: '3.11' + j11_cqlsh_dtests_py3: - <<: *j11_par_executor + <<: *j11_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -2052,7 +2076,7 @@ +@@ -2208,7 +2232,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py38: @@ -305,16 +350,25 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2070,7 +2094,7 @@ +@@ -2226,7 +2250,7 @@ python_version: '3.8' + j11_cqlsh_dtests_py311: +- <<: *j11_par_executor ++ <<: *j11_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -2244,7 +2268,7 @@ + python_version: '3.11' + j8_dtests_large_vnode: - <<: *j8_par_executor + <<: *j8_medium_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -2084,7 +2108,7 @@ +@@ -2258,7 +2282,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests' j8_dtests_large: @@ -323,7 +377,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2098,7 +2122,7 @@ +@@ -2272,7 +2296,7 @@ pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests' j11_dtests_large_vnode: @@ -332,7 +386,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2112,7 +2136,7 @@ +@@ -2286,7 +2310,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests' j11_dtests_large: diff --git a/.circleci/config.yml b/.circleci/config.yml index 323567f960..73d3d0e091 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -295,6 +295,116 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -823,6 +933,117 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_jvm_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1884,6 +2105,116 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4962,6 +5293,117 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j11_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6123,6 +6565,117 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_long_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7564,6 +8117,116 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j8_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -9034,10 +9697,18 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311: + requires: + - start_j8_cqlsh_tests + - j8_build - j8_cqlsh_dtests_py38_vnode: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - start_j8_cqlsh_tests + - j8_build - start_j8_cqlsh_tests_offheap: type: approval - j8_cqlsh_dtests_py3_offheap: @@ -9048,6 +9719,10 @@ workflows: requires: - start_j8_cqlsh_tests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_tests_offheap + - j8_build - start_j11_cqlsh_tests: type: approval - j11_cqlsh_dtests_py3: @@ -9062,10 +9737,18 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j8_build - j11_cqlsh_dtests_py38_vnode: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j8_build - start_j11_cqlsh_tests_offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9076,6 +9759,10 @@ workflows: requires: - start_j11_cqlsh_tests_offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh_tests_offheap + - j8_build java8_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9247,9 +9934,15 @@ workflows: - j8_cqlsh_dtests_py38: requires: - j8_build + - j8_cqlsh_dtests_py311: + requires: + - j8_build - j8_cqlsh_dtests_py38_vnode: requires: - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - j8_build - start_j8_cqlsh_dtests_offheap: type: approval - j8_cqlsh_dtests_py3_offheap: @@ -9260,6 +9953,10 @@ workflows: requires: - start_j8_cqlsh_dtests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_dtests_offheap + - j8_build - j11_cqlsh_dtests_py3: requires: - j8_build @@ -9269,9 +9966,15 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j8_build + - j11_cqlsh_dtests_py311: + requires: + - j8_build - j11_cqlsh_dtests_py38_vnode: requires: - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j8_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9282,6 +9985,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j8_build java11_separate_tests: jobs: - start_j11_build: @@ -9363,10 +10070,18 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j11_build - j11_cqlsh_dtests_py38_vnode: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9377,6 +10092,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_j11_utests_long: type: approval - j11_utests_long: @@ -9468,9 +10187,15 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j11_build + - j11_cqlsh_dtests_py311: + requires: + - j11_build - j11_cqlsh_dtests_py38_vnode: requires: - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9481,6 +10206,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_utests_long: type: approval - j11_utests_long: diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES index 219586716f..d5a573deda 100644 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@ -295,6 +295,116 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -823,6 +933,117 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_jvm_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1884,6 +2105,116 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4962,6 +5293,117 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j11_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6123,6 +6565,117 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_long_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7564,6 +8117,116 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j8_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -9034,10 +9697,18 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311: + requires: + - start_j8_cqlsh_tests + - j8_build - j8_cqlsh_dtests_py38_vnode: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - start_j8_cqlsh_tests + - j8_build - start_j8_cqlsh_tests_offheap: type: approval - j8_cqlsh_dtests_py3_offheap: @@ -9048,6 +9719,10 @@ workflows: requires: - start_j8_cqlsh_tests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_tests_offheap + - j8_build - start_j11_cqlsh_tests: type: approval - j11_cqlsh_dtests_py3: @@ -9062,10 +9737,18 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j8_build - j11_cqlsh_dtests_py38_vnode: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j8_build - start_j11_cqlsh_tests_offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9076,6 +9759,10 @@ workflows: requires: - start_j11_cqlsh_tests_offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh_tests_offheap + - j8_build java8_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9247,9 +9934,15 @@ workflows: - j8_cqlsh_dtests_py38: requires: - j8_build + - j8_cqlsh_dtests_py311: + requires: + - j8_build - j8_cqlsh_dtests_py38_vnode: requires: - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - j8_build - start_j8_cqlsh_dtests_offheap: type: approval - j8_cqlsh_dtests_py3_offheap: @@ -9260,6 +9953,10 @@ workflows: requires: - start_j8_cqlsh_dtests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_dtests_offheap + - j8_build - j11_cqlsh_dtests_py3: requires: - j8_build @@ -9269,9 +9966,15 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j8_build + - j11_cqlsh_dtests_py311: + requires: + - j8_build - j11_cqlsh_dtests_py38_vnode: requires: - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j8_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9282,6 +9985,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j8_build java11_separate_tests: jobs: - start_j11_build: @@ -9363,10 +10070,18 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j11_build - j11_cqlsh_dtests_py38_vnode: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9377,6 +10092,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_j11_utests_long: type: approval - j11_utests_long: @@ -9468,9 +10187,15 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j11_build + - j11_cqlsh_dtests_py311: + requires: + - j11_build - j11_cqlsh_dtests_py38_vnode: requires: - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9481,6 +10206,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_utests_long: type: approval - j11_utests_long: diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES index 323567f960..73d3d0e091 100644 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@ -295,6 +295,116 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -823,6 +933,117 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_jvm_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1884,6 +2105,116 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4962,6 +5293,117 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j11_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6123,6 +6565,117 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_long_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7564,6 +8117,116 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j8_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -9034,10 +9697,18 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311: + requires: + - start_j8_cqlsh_tests + - j8_build - j8_cqlsh_dtests_py38_vnode: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - start_j8_cqlsh_tests + - j8_build - start_j8_cqlsh_tests_offheap: type: approval - j8_cqlsh_dtests_py3_offheap: @@ -9048,6 +9719,10 @@ workflows: requires: - start_j8_cqlsh_tests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_tests_offheap + - j8_build - start_j11_cqlsh_tests: type: approval - j11_cqlsh_dtests_py3: @@ -9062,10 +9737,18 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j8_build - j11_cqlsh_dtests_py38_vnode: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j8_build - start_j11_cqlsh_tests_offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9076,6 +9759,10 @@ workflows: requires: - start_j11_cqlsh_tests_offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh_tests_offheap + - j8_build java8_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9247,9 +9934,15 @@ workflows: - j8_cqlsh_dtests_py38: requires: - j8_build + - j8_cqlsh_dtests_py311: + requires: + - j8_build - j8_cqlsh_dtests_py38_vnode: requires: - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - j8_build - start_j8_cqlsh_dtests_offheap: type: approval - j8_cqlsh_dtests_py3_offheap: @@ -9260,6 +9953,10 @@ workflows: requires: - start_j8_cqlsh_dtests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_dtests_offheap + - j8_build - j11_cqlsh_dtests_py3: requires: - j8_build @@ -9269,9 +9966,15 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j8_build + - j11_cqlsh_dtests_py311: + requires: + - j8_build - j11_cqlsh_dtests_py38_vnode: requires: - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j8_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9282,6 +9985,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j8_build java11_separate_tests: jobs: - start_j11_build: @@ -9363,10 +10070,18 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j11_build - j11_cqlsh_dtests_py38_vnode: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9377,6 +10092,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_j11_utests_long: type: approval - j11_utests_long: @@ -9468,9 +10187,15 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j11_build + - j11_cqlsh_dtests_py311: + requires: + - j11_build - j11_cqlsh_dtests_py38_vnode: requires: - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9481,6 +10206,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_utests_long: type: approval - j11_utests_long: diff --git a/.circleci/config.yml.MIDRES b/.circleci/config.yml.MIDRES index fb535652be..1a073a6865 100644 --- a/.circleci/config.yml.MIDRES +++ b/.circleci/config.yml.MIDRES @@ -295,6 +295,116 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -823,6 +933,117 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_jvm_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1884,6 +2105,116 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j8_cqlsh_dtests_py311: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4962,6 +5293,117 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_dtests_offheap_raw /tmp/all_dtest_tests_j11_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j11_dtests_offheap_raw > /tmp/all_dtest_tests_j11_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_dtests_offheap > /tmp/split_dtest_tests_j11_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j11_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_dtests_offheap_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6123,6 +6565,117 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh_dtests_py311_vnode: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_utests_long_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7564,6 +8117,116 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh_dtests_py311_offheap: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_dtests_offheap) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.11/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n" + - run: + name: Run dtests (j8_dtests_offheap) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt" + cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt + + source ~/env3.11/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.11' ]; then + export CQLSH_PYTHON=/usr/bin/python3.11 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt` + if [ ! -z "$SPLIT_TESTS" ]; then + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + else + echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container" + (exit 1) + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_dtests_offheap + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_dtests_offheap_logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -8810,6 +9473,12 @@ workflows: requires: - start_j8_unit_tests - j8_build + - start_j8_unit_tests_repeat: + type: approval + - j8_unit_tests_repeat: + requires: + - start_j8_unit_tests_repeat + - j8_build - start_j8_jvm_dtests: type: approval - j8_jvm_dtests: @@ -8822,6 +9491,18 @@ workflows: requires: - start_j8_jvm_dtests_vnode - j8_build + - start_j8_jvm_dtests_repeat: + type: approval + - j8_jvm_dtests_repeat: + requires: + - start_j8_jvm_dtests_repeat + - j8_build + - start_j8_jvm_dtests_vnode_repeat: + type: approval + - j8_jvm_dtests_vnode_repeat: + requires: + - start_j8_jvm_dtests_vnode_repeat + - j8_build - start_j11_jvm_dtests: type: approval - j11_jvm_dtests: @@ -8834,12 +9515,30 @@ workflows: requires: - start_j11_jvm_dtests_vnode - j8_build + - start_j11_jvm_dtests_repeat: + type: approval + - j11_jvm_dtests_repeat: + requires: + - start_j11_jvm_dtests_repeat + - j8_build + - start_j11_jvm_dtests_vnode_repeat: + type: approval + - j11_jvm_dtests_vnode_repeat: + requires: + - start_j11_jvm_dtests_vnode_repeat + - j8_build - start_j8_simulator_dtests: type: approval - j8_simulator_dtests: requires: - start_j8_simulator_dtests - j8_build + - start_j8_simulator_dtests_repeat: + type: approval + - j8_simulator_dtests_repeat: + requires: + - start_j8_simulator_dtests_repeat + - j8_build - start_j8_cqlshlib_tests: type: approval - j8_cqlshlib_tests: @@ -8870,6 +9569,12 @@ workflows: requires: - start_j11_unit_tests - j8_build + - start_j11_unit_tests_repeat: + type: approval + - j11_unit_tests_repeat: + requires: + - start_j11_unit_tests_repeat + - j8_build - start_j8_utests_long: type: approval - j8_utests_long: @@ -8882,6 +9587,18 @@ workflows: requires: - start_j11_utests_long - j8_build + - start_j8_utests_long_repeat: + type: approval + - j8_utests_long_repeat: + requires: + - start_j8_utests_long_repeat + - j8_build + - start_j11_utests_long_repeat: + type: approval + - j11_utests_long_repeat: + requires: + - start_j11_utests_long_repeat + - j8_build - start_j8_utests_cdc: type: approval - j8_utests_cdc: @@ -8894,6 +9611,18 @@ workflows: requires: - start_j11_utests_cdc - j8_build + - start_j8_utests_cdc_repeat: + type: approval + - j8_utests_cdc_repeat: + requires: + - start_j8_utests_cdc_repeat + - j8_build + - start_j11_utests_cdc_repeat: + type: approval + - j11_utests_cdc_repeat: + requires: + - start_j11_utests_cdc_repeat + - j8_build - start_j8_utests_compression: type: approval - j8_utests_compression: @@ -8906,6 +9635,18 @@ workflows: requires: - start_j11_utests_compression - j8_build + - start_j8_utests_compression_repeat: + type: approval + - j8_utests_compression_repeat: + requires: + - start_j8_utests_compression_repeat + - j8_build + - start_j11_utests_compression_repeat: + type: approval + - j11_utests_compression_repeat: + requires: + - start_j11_utests_compression_repeat + - j8_build - start_j8_utests_stress: type: approval - j8_utests_stress: @@ -8918,6 +9659,18 @@ workflows: requires: - start_j11_utests_stress - j8_build + - start_j8_utests_stress_repeat: + type: approval + - j8_utests_stress_repeat: + requires: + - start_j8_utests_stress_repeat + - j8_build + - start_j11_utests_stress_repeat: + type: approval + - j11_utests_stress_repeat: + requires: + - start_j11_utests_stress_repeat + - j8_build - start_j8_utests_fqltool: type: approval - j8_utests_fqltool: @@ -8930,6 +9683,18 @@ workflows: requires: - start_j11_utests_fqltool - j8_build + - start_j8_utests_fqltool_repeat: + type: approval + - j8_utests_fqltool_repeat: + requires: + - start_j8_utests_fqltool_repeat + - j8_build + - start_j11_utests_fqltool_repeat: + type: approval + - j11_utests_fqltool_repeat: + requires: + - start_j11_utests_fqltool_repeat + - j8_build - start_j8_utests_system_keyspace_directory: type: approval - j8_utests_system_keyspace_directory: @@ -8942,6 +9707,18 @@ workflows: requires: - start_j11_utests_system_keyspace_directory - j8_build + - start_j8_utests_system_keyspace_directory_repeat: + type: approval + - j8_utests_system_keyspace_directory_repeat: + requires: + - start_j8_utests_system_keyspace_directory_repeat + - j8_build + - start_j11_utests_system_keyspace_directory_repeat: + type: approval + - j11_utests_system_keyspace_directory_repeat: + requires: + - start_j11_utests_system_keyspace_directory_repeat + - j8_build - start_j8_dtest_jars_build: type: approval - j8_dtest_jars_build: @@ -8954,6 +9731,12 @@ workflows: requires: - start_jvm_upgrade_dtests - j8_dtest_jars_build + - start_jvm_upgrade_dtests_repeat: + type: approval + - j8_jvm_upgrade_dtests_repeat: + requires: + - start_jvm_upgrade_dtests_repeat + - j8_dtest_jars_build - start_j8_dtests: type: approval - j8_dtests: @@ -8972,6 +9755,12 @@ workflows: requires: - start_j8_dtests_offheap - j8_build + - start_j8_dtests_offheap_repeat: + type: approval + - j8_dtests_offheap_repeat: + requires: + - start_j8_dtests_offheap_repeat + - j8_build - start_j11_dtests: type: approval - j11_dtests: @@ -8990,30 +9779,60 @@ workflows: requires: - start_j11_dtests_offheap - j8_build + - start_j11_dtests_offheap_repeat: + type: approval + - j11_dtests_offheap_repeat: + requires: + - start_j11_dtests_offheap_repeat + - j8_build - start_j8_dtests_large: type: approval - j8_dtests_large: requires: - start_j8_dtests_large - j8_build + - start_j8_dtests_large_repeat: + type: approval + - j8_dtests_large_repeat: + requires: + - start_j8_dtests_large_repeat + - j8_build - start_j8_dtests_large_vnode: type: approval - j8_dtests_large_vnode: requires: - start_j8_dtests_large_vnode - j8_build + - start_j8_dtests_large_vnode_repeat: + type: approval + - j8_dtests_large_vnode_repeat: + requires: + - start_j8_dtests_large_vnode_repeat + - j8_build - start_j11_dtests_large: type: approval - j11_dtests_large: requires: - start_j11_dtests_large - j8_build + - start_j11_dtests_large_repeat: + type: approval + - j11_dtests_large_repeat: + requires: + - start_j11_dtests_large_repeat + - j8_build - start_j11_dtests_large_vnode: type: approval - j11_dtests_large_vnode: requires: - start_j11_dtests_large_vnode - j8_build + - start_j11_dtests_large_vnode_repeat: + type: approval + - j11_dtests_large_vnode_repeat: + requires: + - start_j11_dtests_large_vnode_repeat + - j8_build - start_upgrade_dtests: type: approval - j8_upgrade_dtests: @@ -9034,10 +9853,18 @@ workflows: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311: + requires: + - start_j8_cqlsh_tests + - j8_build - j8_cqlsh_dtests_py38_vnode: requires: - start_j8_cqlsh_tests - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - start_j8_cqlsh_tests + - j8_build - start_j8_cqlsh_tests_offheap: type: approval - j8_cqlsh_dtests_py3_offheap: @@ -9048,6 +9875,10 @@ workflows: requires: - start_j8_cqlsh_tests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_tests_offheap + - j8_build - start_j11_cqlsh_tests: type: approval - j11_cqlsh_dtests_py3: @@ -9062,10 +9893,18 @@ workflows: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j8_build - j11_cqlsh_dtests_py38_vnode: requires: - start_j11_cqlsh_tests - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j8_build - start_j11_cqlsh_tests_offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9076,6 +9915,52 @@ workflows: requires: - start_j11_cqlsh_tests_offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh_tests_offheap + - j8_build + - start_j8_repeated_ant_test: + type: approval + - j8_repeated_ant_test: + requires: + - start_j8_repeated_ant_test + - j8_build + - start_j11_repeated_ant_test: + type: approval + - j11_repeated_ant_test: + requires: + - start_j11_repeated_ant_test + - j8_build + - start_j8_dtests_repeat: + type: approval + - j8_dtests_repeat: + requires: + - start_j8_dtests_repeat + - j8_build + - start_j8_dtests_vnode_repeat: + type: approval + - j8_dtests_vnode_repeat: + requires: + - start_j8_dtests_vnode_repeat + - j8_build + - start_j11_dtests_repeat: + type: approval + - j11_dtests_repeat: + requires: + - start_j11_dtests_repeat + - j8_build + - start_j11_dtests_vnode_repeat: + type: approval + - j11_dtests_vnode_repeat: + requires: + - start_j11_dtests_vnode_repeat + - j8_build + - start_j8_upgrade_dtests_repeat: + type: approval + - j8_upgrade_dtests_repeat: + requires: + - start_j8_upgrade_dtests_repeat + - j8_build java8_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9086,21 +9971,39 @@ workflows: - j8_unit_tests: requires: - j8_build + - j8_unit_tests_repeat: + requires: + - j8_build - j8_simulator_dtests: requires: - j8_build + - j8_simulator_dtests_repeat: + requires: + - j8_build - j8_jvm_dtests: requires: - j8_build + - j8_jvm_dtests_repeat: + requires: + - j8_build - j8_jvm_dtests_vnode: requires: - j8_build + - j8_jvm_dtests_vnode_repeat: + requires: + - j8_build - j11_jvm_dtests: requires: - j8_build + - j11_jvm_dtests_repeat: + requires: + - j8_build - j11_jvm_dtests_vnode: requires: - j8_build + - j11_jvm_dtests_vnode_repeat: + requires: + - j8_build - j8_cqlshlib_tests: requires: - j8_build @@ -9116,6 +10019,9 @@ workflows: - j11_unit_tests: requires: - j8_build + - j11_unit_tests_repeat: + requires: + - j8_build - start_utests_long: type: approval - j8_utests_long: @@ -9126,6 +10032,14 @@ workflows: requires: - start_utests_long - j8_build + - j8_utests_long_repeat: + requires: + - start_utests_long + - j8_build + - j11_utests_long_repeat: + requires: + - start_utests_long + - j8_build - start_utests_cdc: type: approval - j8_utests_cdc: @@ -9136,6 +10050,14 @@ workflows: requires: - start_utests_cdc - j8_build + - j8_utests_cdc_repeat: + requires: + - start_utests_cdc + - j8_build + - j11_utests_cdc_repeat: + requires: + - start_utests_cdc + - j8_build - start_utests_compression: type: approval - j8_utests_compression: @@ -9146,6 +10068,14 @@ workflows: requires: - start_utests_compression - j8_build + - j8_utests_compression_repeat: + requires: + - start_utests_compression + - j8_build + - j11_utests_compression_repeat: + requires: + - start_utests_compression + - j8_build - start_utests_stress: type: approval - j8_utests_stress: @@ -9156,6 +10086,14 @@ workflows: requires: - start_utests_stress - j8_build + - j8_utests_stress_repeat: + requires: + - start_utests_stress + - j8_build + - j11_utests_stress_repeat: + requires: + - start_utests_stress + - j8_build - start_utests_fqltool: type: approval - j8_utests_fqltool: @@ -9166,6 +10104,14 @@ workflows: requires: - start_utests_fqltool - j8_build + - j8_utests_fqltool_repeat: + requires: + - start_utests_fqltool + - j8_build + - j11_utests_fqltool_repeat: + requires: + - start_utests_fqltool + - j8_build - start_utests_system_keyspace_directory: type: approval - j8_utests_system_keyspace_directory: @@ -9175,6 +10121,13 @@ workflows: requires: - start_utests_system_keyspace_directory - j8_build + - j8_utests_system_keyspace_directory_repeat: + requires: + - j8_build + - j11_utests_system_keyspace_directory_repeat: + requires: + - start_utests_system_keyspace_directory + - j8_build - start_jvm_upgrade_dtests: type: approval - j8_dtest_jars_build: @@ -9184,60 +10137,111 @@ workflows: - j8_jvm_upgrade_dtests: requires: - j8_dtest_jars_build + - j8_jvm_upgrade_dtests_repeat: + requires: + - j8_dtest_jars_build - j8_dtests: requires: - j8_build + - j8_dtests_repeat: + requires: + - j8_build - j8_dtests_vnode: requires: - j8_build + - j8_dtests_vnode_repeat: + requires: + - j8_build - start_j8_dtests_offheap: type: approval - j8_dtests_offheap: requires: - start_j8_dtests_offheap - j8_build + - j8_dtests_offheap_repeat: + requires: + - start_j8_dtests_offheap + - j8_build - j11_dtests: requires: - j8_build + - j11_dtests_repeat: + requires: + - j8_build - j11_dtests_vnode: requires: - j8_build + - j11_dtests_vnode_repeat: + requires: + - j8_build - start_j11_dtests_offheap: type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j8_build + - j11_dtests_offheap_repeat: + requires: + - start_j11_dtests_offheap + - j8_build - start_j8_dtests_large: type: approval - j8_dtests_large: requires: - start_j8_dtests_large - j8_build + - start_j8_dtests_large_repeat: + type: approval + - j8_dtests_large_repeat: + requires: + - start_j8_dtests_large_repeat + - j8_build - start_j8_dtests_large_vnode: type: approval - j8_dtests_large_vnode: requires: - start_j8_dtests_large_vnode - j8_build + - start_j8_dtests_large_vnode_repeat: + type: approval + - j8_dtests_large_vnode_repeat: + requires: + - start_j8_dtests_large_vnode_repeat + - j8_build - start_j11_dtests_large: type: approval - j11_dtests_large: requires: - start_j11_dtests_large - j8_build + - start_j11_dtests_large_repeat: + type: approval + - j11_dtests_large_repeat: + requires: + - start_j11_dtests_large_repeat + - j8_build - start_j11_dtests_large_vnode: type: approval - j11_dtests_large_vnode: requires: - start_j11_dtests_large_vnode - j8_build + - start_j11_dtests_large_vnode_repeat: + type: approval + - j11_dtests_large_vnode_repeat: + requires: + - start_j11_dtests_large_vnode_repeat + - j8_build - start_upgrade_dtests: type: approval - j8_upgrade_dtests: requires: - j8_build - start_upgrade_dtests + - j8_upgrade_dtests_repeat: + requires: + - j8_build + - start_upgrade_dtests - j8_cqlsh_dtests_py3: requires: - j8_build @@ -9247,9 +10251,15 @@ workflows: - j8_cqlsh_dtests_py38: requires: - j8_build + - j8_cqlsh_dtests_py311: + requires: + - j8_build - j8_cqlsh_dtests_py38_vnode: requires: - j8_build + - j8_cqlsh_dtests_py311_vnode: + requires: + - j8_build - start_j8_cqlsh_dtests_offheap: type: approval - j8_cqlsh_dtests_py3_offheap: @@ -9260,6 +10270,10 @@ workflows: requires: - start_j8_cqlsh_dtests_offheap - j8_build + - j8_cqlsh_dtests_py311_offheap: + requires: + - start_j8_cqlsh_dtests_offheap + - j8_build - j11_cqlsh_dtests_py3: requires: - j8_build @@ -9269,9 +10283,15 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j8_build + - j11_cqlsh_dtests_py311: + requires: + - j8_build - j11_cqlsh_dtests_py38_vnode: requires: - j8_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j8_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9282,6 +10302,10 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j8_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j8_build java11_separate_tests: jobs: - start_j11_build: @@ -9295,6 +10319,12 @@ workflows: requires: - start_j11_unit_tests - j11_build + - start_j11_unit_tests_repeat: + type: approval + - j11_unit_tests_repeat: + requires: + - start_j11_unit_tests_repeat + - j11_build - start_j11_jvm_dtests: type: approval - j11_jvm_dtests: @@ -9307,6 +10337,18 @@ workflows: requires: - start_j11_jvm_dtests_vnode - j11_build + - start_j11_jvm_dtests_repeat: + type: approval + - j11_jvm_dtests_repeat: + requires: + - start_j11_jvm_dtests_repeat + - j11_build + - start_j11_jvm_dtests_vnode_repeat: + type: approval + - j11_jvm_dtests_vnode_repeat: + requires: + - start_j11_jvm_dtests_vnode_repeat + - j11_build - start_j11_cqlshlib_tests: type: approval - j11_cqlshlib_tests: @@ -9337,18 +10379,36 @@ workflows: requires: - start_j11_dtests_offheap - j11_build + - start_j11_dtests_offheap_repeat: + type: approval + - j11_dtests_offheap_repeat: + requires: + - start_j11_dtests_offheap_repeat + - j11_build - start_j11_dtests_large: type: approval - j11_dtests_large: requires: - start_j11_dtests_large - j11_build + - start_j11_dtests_large_repeat: + type: approval + - j11_dtests_large_repeat: + requires: + - start_j11_dtests_large_repeat + - j11_build - start_j11_dtests_large_vnode: type: approval - j11_dtests_large_vnode: requires: - start_j11_dtests_large_vnode - j11_build + - start_j11_dtests_large_vnode_repeat: + type: approval + - j11_dtests_large_vnode_repeat: + requires: + - start_j11_dtests_large_vnode_repeat + - j11_build - start_j11_cqlsh_tests: type: approval - j11_cqlsh_dtests_py3: @@ -9363,10 +10423,18 @@ workflows: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311: + requires: + - start_j11_cqlsh_tests + - j11_build - j11_cqlsh_dtests_py38_vnode: requires: - start_j11_cqlsh_tests - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - start_j11_cqlsh_tests + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9377,42 +10445,100 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_j11_utests_long: type: approval - j11_utests_long: requires: - start_j11_utests_long - j11_build + - start_j11_utests_long_repeat: + type: approval + - j11_utests_long_repeat: + requires: + - start_j11_utests_long_repeat + - j11_build - start_j11_utests_cdc: type: approval - j11_utests_cdc: requires: - start_j11_utests_cdc - j11_build + - start_j11_utests_cdc_repeat: + type: approval + - j11_utests_cdc_repeat: + requires: + - start_j11_utests_cdc_repeat + - j11_build - start_j11_utests_compression: type: approval - j11_utests_compression: requires: - start_j11_utests_compression - j11_build + - start_j11_utests_compression_repeat: + type: approval + - j11_utests_compression_repeat: + requires: + - start_j11_utests_compression_repeat + - j11_build - start_j11_utests_stress: type: approval - j11_utests_stress: requires: - start_j11_utests_stress - j11_build + - start_j11_utests_stress_repeat: + type: approval + - j11_utests_stress_repeat: + requires: + - start_j11_utests_stress_repeat + - j11_build - start_j11_utests_fqltool: type: approval - j11_utests_fqltool: requires: - start_j11_utests_fqltool - j11_build + - start_j11_utests_fqltool_repeat: + type: approval + - j11_utests_fqltool_repeat: + requires: + - start_j11_utests_fqltool_repeat + - j11_build - start_j11_utests_system_keyspace_directory: type: approval - j11_utests_system_keyspace_directory: requires: - start_j11_utests_system_keyspace_directory - j11_build + - start_j11_utests_system_keyspace_directory_repeat: + type: approval + - j11_utests_system_keyspace_directory_repeat: + requires: + - start_j11_utests_system_keyspace_directory_repeat + - j11_build + - start_j11_repeated_ant_test: + type: approval + - j11_repeated_ant_test: + requires: + - start_j11_repeated_ant_test + - j11_build + - start_j11_dtests_repeat: + type: approval + - j11_dtests_repeat: + requires: + - start_j11_dtests_repeat + - j11_build + - start_j11_dtests_vnode_repeat: + type: approval + - j11_dtests_vnode_repeat: + requires: + - start_j11_dtests_vnode_repeat + - j11_build java11_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9423,12 +10549,21 @@ workflows: - j11_unit_tests: requires: - j11_build + - j11_unit_tests_repeat: + requires: + - j11_build - j11_jvm_dtests: requires: - j11_build + - j11_jvm_dtests_repeat: + requires: + - j11_build - j11_jvm_dtests_vnode: requires: - j11_build + - j11_jvm_dtests_vnode_repeat: + requires: + - j11_build - j11_cqlshlib_tests: requires: - j11_build @@ -9438,27 +10573,51 @@ workflows: - j11_dtests: requires: - j11_build + - j11_dtests_repeat: + requires: + - j11_build - j11_dtests_vnode: requires: - j11_build + - j11_dtests_vnode_repeat: + requires: + - j11_build - start_j11_dtests_offheap: type: approval - j11_dtests_offheap: requires: - start_j11_dtests_offheap - j11_build + - start_j11_dtests_offheap_repeat: + type: approval + - j11_dtests_offheap_repeat: + requires: + - start_j11_dtests_offheap_repeat + - j11_build - start_j11_dtests_large: type: approval - j11_dtests_large: requires: - start_j11_dtests_large - j11_build + - start_j11_dtests_large_repeat: + type: approval + - j11_dtests_large_repeat: + requires: + - start_j11_dtests_large_repeat + - j11_build - start_j11_dtests_large_vnode: type: approval - j11_dtests_large_vnode: requires: - start_j11_dtests_large_vnode - j11_build + - start_j11_dtests_large_vnode_repeat: + type: approval + - j11_dtests_large_vnode_repeat: + requires: + - start_j11_dtests_large_vnode_repeat + - j11_build - j11_cqlsh_dtests_py3: requires: - j11_build @@ -9468,9 +10627,15 @@ workflows: - j11_cqlsh_dtests_py38: requires: - j11_build + - j11_cqlsh_dtests_py311: + requires: + - j11_build - j11_cqlsh_dtests_py38_vnode: requires: - j11_build + - j11_cqlsh_dtests_py311_vnode: + requires: + - j11_build - start_j11_cqlsh-dtests-offheap: type: approval - j11_cqlsh_dtests_py3_offheap: @@ -9481,39 +10646,67 @@ workflows: requires: - start_j11_cqlsh-dtests-offheap - j11_build + - j11_cqlsh_dtests_py311_offheap: + requires: + - start_j11_cqlsh-dtests-offheap + - j11_build - start_utests_long: type: approval - j11_utests_long: requires: - start_utests_long - j11_build + - j11_utests_long_repeat: + requires: + - start_utests_long + - j11_build - start_utests_cdc: type: approval - j11_utests_cdc: requires: - start_utests_cdc - j11_build + - j11_utests_cdc_repeat: + requires: + - start_utests_cdc + - j11_build - start_utests_compression: type: approval - j11_utests_compression: requires: - start_utests_compression - j11_build + - j11_utests_compression_repeat: + requires: + - start_utests_compression + - j11_build - start_utests_stress: type: approval - j11_utests_stress: requires: - start_utests_stress - j11_build + - j11_utests_stress_repeat: + requires: + - start_utests_stress + - j11_build - start_utests_fqltool: type: approval - j11_utests_fqltool: requires: - start_utests_fqltool - j11_build + - j11_utests_fqltool_repeat: + requires: + - start_utests_fqltool + - j11_build - start_utests_system_keyspace_directory: type: approval - j11_utests_system_keyspace_directory: requires: - start_utests_system_keyspace_directory - j11_build + - j11_utests_system_keyspace_directory_repeat: + requires: + - start_utests_system_keyspace_directory + - j11_build From 4f32d43d88a36a5dbf0ae1d42bb0280805c99649 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Fri, 16 Dec 2022 10:09:39 -0600 Subject: [PATCH 08/10] get newest pip in cassandra-cqlsh-tests.sh Patch by brandonwilliams; reviewed by bereng for CASSANDRA-18121 --- pylib/cassandra-cqlsh-tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh index 951f69a61a..ebdba4efaa 100755 --- a/pylib/cassandra-cqlsh-tests.sh +++ b/pylib/cassandra-cqlsh-tests.sh @@ -73,6 +73,8 @@ fi set -e # enable immediate exit if venv setup fails virtualenv --python=$PYTHON_VERSION venv source venv/bin/activate +# 3.11 needs the newest pip +curl -sS https://bootstrap.pypa.io/get-pip.py | $PYTHON_VERSION pip install -r ${CASSANDRA_DIR}/pylib/requirements.txt pip freeze From 5086b7d0a7378fa6648d85900776e336a765361c Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Tue, 17 Jan 2023 14:18:56 -0600 Subject: [PATCH 09/10] Upgrade cython Patch by brandonwilliams; reviewed by bereng for CASSANDRA-18121 --- pylib/cassandra-cqlsh-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh index ebdba4efaa..ad6ae75368 100755 --- a/pylib/cassandra-cqlsh-tests.sh +++ b/pylib/cassandra-cqlsh-tests.sh @@ -81,7 +81,7 @@ pip freeze if [ "$cython" = "yes" ]; then TESTSUITE_NAME="${TESTSUITE_NAME}.cython" - pip install "Cython>=0.27.2,<0.28" + pip install "Cython>=0.29.15,<3.0" cd pylib/; python setup.py build_ext --inplace cd ${WORKSPACE} else From 9a0af4112e87f5b97056aa39e63c5ab461b60237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20de=20la=20Pe=C3=B1a?= Date: Mon, 9 Jan 2023 13:50:47 +0000 Subject: [PATCH 10/10] Fix legacy clustering serialization for paging with compact storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit patch by Andrés de la Peña; reviewed by Berenguer Blasi and Piotr Kołaczkowski for CASSANDRA-17507 --- CHANGES.txt | 1 + NEWS.txt | 14 ++ .../cassandra/service/pager/PagingState.java | 8 + ...ompactStoragePagingWithProtocolTester.java | 179 ++++++++++++++++++ ...mpactStoragePagingWithProtocolV30Test.java | 33 ++++ ...mpactStoragePagingWithProtocolV3XTest.java | 33 ++++ 6 files changed, 268 insertions(+) create mode 100644 test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolTester.java create mode 100644 test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV30Test.java create mode 100644 test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV3XTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 8794ae533a..f032fc390e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0.8 + * Fix legacy clustering serialization for paging with compact storage (CASSANDRA-17507) * Add support for python 3.11 (CASSANDRA-18088) * Fix formatting of duration in cqlsh (CASSANDRA-18141) * Fix sstable loading of keyspaces named snapshots or backups (CASSANDRA-14013) diff --git a/NEWS.txt b/NEWS.txt index 7d6bf91cf9..4915652351 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -51,6 +51,20 @@ restore snapshots created with the previous major version using the 'sstableloader' tool. You can upgrade the file format of your snapshots using the provided 'sstableupgrade' tool. +4.0.8 +===== + +New features +------------ + +Upgrading +--------- + - All previous versions of 4.x contained a mistake on the implementation of the old CQL native protocol v3. That + mistake produced issues when paging over tables with compact storage and a single clustering column during rolling + upgrades involving 3.x and 4.x nodes. The fix for that issue makes it can now appear during rolling upgrades from + 4.0.0-4.0.7. If that is your case, please use protocol v4 or higher in your driver. See CASSANDRA-17507 for further + details. + 4.0.6 ===== diff --git a/src/java/org/apache/cassandra/service/pager/PagingState.java b/src/java/org/apache/cassandra/service/pager/PagingState.java index 2b160329de..2c2b08bf72 100644 --- a/src/java/org/apache/cassandra/service/pager/PagingState.java +++ b/src/java/org/apache/cassandra/service/pager/PagingState.java @@ -422,6 +422,10 @@ public class PagingState // Old (pre-3.0) encoding of cells. We need that for the protocol v3 as that is how things where encoded private static ByteBuffer encodeCellName(TableMetadata metadata, Clustering clustering, ByteBuffer columnName, ByteBuffer collectionElement) { + // v30 and v3X don't use composites for single-element clusterings in compact tables + if (metadata.isCompactTable() && metadata.comparator.size() == 1) + return clustering.bufferAt(0); + boolean isStatic = clustering == Clustering.STATIC_CLUSTERING; // We use comparator.size() rather than clustering.size() because of static clusterings @@ -458,6 +462,10 @@ public class PagingState if (csize == 0) return Clustering.EMPTY; + // v30 and v3X don't use composites for single-element clusterings in compact tables + if (metadata.isCompactTable() && metadata.comparator.size() == 1) + return Clustering.make(value); + if (CompositeType.isStaticName(value, ByteBufferAccessor.instance)) return Clustering.STATIC_CLUSTERING; diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolTester.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolTester.java new file mode 100644 index 0000000000..2683e7d4d3 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolTester.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.upgrade; + +import java.nio.ByteBuffer; +import java.util.List; + +import org.junit.Test; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.ColumnDefinitions; +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.ProtocolVersion; +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Row; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SimpleStatement; +import com.datastax.driver.core.Statement; +import com.vdurmont.semver4j.Semver; +import org.apache.cassandra.distributed.api.ConsistencyLevel; + +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL; +import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.junit.Assert.assertEquals; + +/** + * Tests paging over a table with {@code COMPACT STORAGE} in a mixed version cluster using different protocol versions. + */ +public abstract class CompactStoragePagingWithProtocolTester extends UpgradeTestBase +{ + /** + * The initial version from which we are upgrading. + */ + protected abstract Semver initialVersion(); + + @Test + public void testPagingWithCompactStorageSingleClustering() throws Throwable + { + Object[] row1 = new Object[]{ "0", "01", "v" }; + Object[] row2 = new Object[]{ "0", "02", "v" }; + Object[] row3 = new Object[]{ "1", "01", "v" }; + Object[] row4 = new Object[]{ "1", "02", "v" }; + + new TestCase() + .nodes(2) + .nodesToUpgrade(1) + .singleUpgrade(initialVersion(), CURRENT) + .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL)) + .setup(c -> { + c.schemaChange(withKeyspace("CREATE TABLE %s.t (pk text, ck text, v text, " + + "PRIMARY KEY (pk, ck)) WITH COMPACT STORAGE")); + String insert = withKeyspace("INSERT INTO %s.t (pk, ck, v) VALUES (?, ?, ?)"); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row1); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row2); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row3); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row4); + }) + .runAfterNodeUpgrade((cluster, node) -> assertRowsWithAllProtocolVersions(row1, row2, row3, row4)) + .run(); + } + + @Test + public void testPagingWithCompactStorageMultipleClusterings() throws Throwable + { + Object[] row1 = new Object[]{ "0", "01", "10", "v" }; + Object[] row2 = new Object[]{ "0", "01", "20", "v" }; + Object[] row3 = new Object[]{ "0", "02", "10", "v" }; + Object[] row4 = new Object[]{ "0", "02", "20", "v" }; + Object[] row5 = new Object[]{ "1", "01", "10", "v" }; + + new TestCase() + .nodes(2) + .nodesToUpgrade(1) + .singleUpgrade(initialVersion(), CURRENT) + .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL)) + .setup(c -> { + c.schemaChange(withKeyspace("CREATE TABLE %s.t (pk text, ck1 text, ck2 text, v text, " + + "PRIMARY KEY (pk, ck1, ck2)) WITH COMPACT STORAGE")); + String insert = withKeyspace("INSERT INTO %s.t (pk, ck1, ck2, v) VALUES (?, ?, ?, ?)"); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row1); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row2); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row3); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row4); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row5); + }) + .runAfterNodeUpgrade((cluster, node) -> assertRowsWithAllProtocolVersions(row1, row2, row3, row4, row5)) + .run(); + } + + @Test + public void testPagingWithCompactStorageWithoutClustering() throws Throwable + { + Object[] row1 = new Object[]{ "1", "v1", "v2" }; + Object[] row2 = new Object[]{ "2", "v1", "v2" }; + Object[] row3 = new Object[]{ "3", "v1", "v2" }; + + new TestCase() + .nodes(2) + .nodesToUpgrade(1) + .singleUpgrade(initialVersion(), CURRENT) + .withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL)) + .setup(c -> { + c.schemaChange(withKeyspace("CREATE TABLE %s.t (pk text PRIMARY KEY, v1 text, v2 text) WITH COMPACT STORAGE")); + String insert = withKeyspace("INSERT INTO %s.t (pk, v1, v2) VALUES (?, ?, ?)"); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row1); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row2); + c.coordinator(1).execute(insert, ConsistencyLevel.ALL, row3); + }) + .runAfterNodeUpgrade((cluster, node) -> assertRowsWithAllProtocolVersions(row3, row2, row1)) + .run(); + } + + private void assertRowsWithAllProtocolVersions(Object[]... rows) + { + String query = withKeyspace("SELECT * FROM %s.t"); + assertRows(query, ProtocolVersion.V3, rows); + assertRows(query, ProtocolVersion.V4, rows); + if (initialVersion().isGreaterThanOrEqualTo(v3X)) + assertRows(query, ProtocolVersion.V5, rows); + } + + private static void assertRows(String query, ProtocolVersion protocolVersion, Object[]... expectedRows) + { + Cluster.Builder builder = com.datastax.driver.core.Cluster.builder() + .addContactPoint("127.0.0.1") + .withProtocolVersion(protocolVersion); + try (com.datastax.driver.core.Cluster cluster = builder.build(); + Session session = cluster.connect()) + { + Statement stmt = new SimpleStatement(query); + stmt.setConsistencyLevel(com.datastax.driver.core.ConsistencyLevel.ALL); + stmt.setFetchSize(1); + + ResultSet result = session.execute(stmt); + List actualRows = result.all(); + assertEquals(expectedRows.length, actualRows.size()); + + ColumnDefinitions columnDefs = result.getColumnDefinitions(); + com.datastax.driver.core.ProtocolVersion driverProtocolVersion = + com.datastax.driver.core.ProtocolVersion.fromInt(protocolVersion.toInt()); + + for (int rowIndex = 0; rowIndex < expectedRows.length; rowIndex++) + { + Object[] expectedRow = expectedRows[rowIndex]; + Row actualRow = actualRows.get(rowIndex); + + assertEquals(expectedRow.length, actualRow.getColumnDefinitions().size()); + + for (int columnIndex = 0; columnIndex < columnDefs.size(); columnIndex++) + { + DataType type = columnDefs.getType(columnIndex); + ByteBuffer expectedByteValue = cluster.getConfiguration() + .getCodecRegistry() + .codecFor(type) + .serialize(expectedRow[columnIndex], driverProtocolVersion); + ByteBuffer actualValue = actualRow.getBytesUnsafe(columnDefs.getName(columnIndex)); + assertEquals(expectedByteValue, actualValue); + } + } + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV30Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV30Test.java new file mode 100644 index 0000000000..518a514146 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV30Test.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.upgrade; + +import com.vdurmont.semver4j.Semver; + +/** + * {@link CompactStoragePagingWithProtocolTester} for v30 -> CURRENT upgrade path. + */ +public class CompactStoragePagingWithProtocolV30Test extends CompactStoragePagingWithProtocolTester +{ + @Override + protected Semver initialVersion() + { + return v30; + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV3XTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV3XTest.java new file mode 100644 index 0000000000..003c372ceb --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/CompactStoragePagingWithProtocolV3XTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.upgrade; + +import com.vdurmont.semver4j.Semver; + +/** + * {@link CompactStoragePagingWithProtocolTester} for v3X -> CURRENT upgrade path. + */ +public class CompactStoragePagingWithProtocolV3XTest extends CompactStoragePagingWithProtocolTester +{ + @Override + protected Semver initialVersion() + { + return v3X; + } +}