natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-3.0' into cassandra-3.11

This commit is contained in:
Ekaterina Dimitrova 2023-02-11 14:32:37 -05:00
parent 74b0b2086f 476d31bce2
commit 788b4eca2d
9 changed files with 2881 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

204
.circleci/config-2_1.yml
View File

@ -96,6 +96,15 @@ default_env_vars: &default_env_vars
# The number of times that the manually specified Python dtests should be run.
REPEATED_DTESTS_COUNT: 500
# Comma-separated list of Python large dtests that should be repeatedly run, for example:
# REPEATED_LARGE_DTESTS: replace_address_test.py
# REPEATED_LARGE_DTESTS: replace_address_test.py::TestReplaceAddress
# REPEATED_LARGE_DTESTS: replace_address_test.py::TestReplaceAddress::test_replace_stopped_node
# REPEATED_LARGE_DTESTS: replace_address_test.py,materialized_views_test.py
REPEATED_LARGE_DTESTS:
# The number of times that the manually specified Python large dtests should be run.
REPEATED_LARGE_DTESTS_COUNT: 100
# Comma-separated list of Python dtests that should be repeatedly run, for example:
# REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py
# REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py::TestCQLNodes2RF1_Upgrade_current_4_0_x_To_indev_4_0_x
@ -134,6 +143,12 @@ j8_small_par_executor: &j8_small_par_executor
#exec_resource_class: xlarge
parallelism: 1
j8_small_executor: &j8_small_executor
executor:
name: java8-executor
exec_resource_class: medium
parallelism: 1
j8_medium_par_executor: &j8_medium_par_executor
executor:
name: java8-executor
@ -198,6 +213,18 @@ separate_jobs: &separate_jobs
requires:
- start_j8_jvm_dtests_repeat
- build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
requires:
- start_j8_cqlshlib_tests
- build
- start_j8_cqlshlib_cython_tests:
type: approval
- j8_cqlshlib_cython_tests:
requires:
- start_j8_cqlshlib_cython_tests
- build
# specialized unit tests (all run on request)
- start_utests_long:
type: approval
@ -265,7 +292,7 @@ separate_jobs: &separate_jobs
requires:
- start_jvm_upgrade_dtests_repeat
- j8_dtest_jars_build
# Java 8 dtests
# Python DTests
- start_j8_dtests:
type: approval
- j8_dtests:
@ -290,6 +317,31 @@ separate_jobs: &separate_jobs
requires:
- start_j8_dtests_vnode_repeat
- build
# Python large DTests
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- start_j8_dtests_large_repeat:
type: approval
- j8_dtests_large_repeat:
requires:
- start_j8_dtests_large_repeat
- build
- start_j8_dtests_large_vnode:
type: approval
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large_vnode
- build
- start_j8_dtests_large_vnode_repeat:
type: approval
- j8_dtests_large_vnode_repeat:
requires:
- start_j8_dtests_large_vnode_repeat
- build
# Java 8 upgrade tests
- start_upgrade_dtests:
type: approval
@ -344,6 +396,12 @@ pre-commit_jobs: &pre-commit_jobs
- j8_jvm_dtests_repeat:
requires:
- build
- j8_cqlshlib_tests:
requires:
- build
- j8_cqlshlib_cython_tests:
requires:
- build
# specialized unit tests (all run on request)
- start_utests_long:
type: approval
@ -397,7 +455,7 @@ pre-commit_jobs: &pre-commit_jobs
- j8_jvm_upgrade_dtests_repeat:
requires:
- j8_dtest_jars_build
# Java 8 dtests
# Python DTests
- j8_dtests:
requires:
- build
@ -410,6 +468,25 @@ pre-commit_jobs: &pre-commit_jobs
- j8_dtests_vnode_repeat:
requires:
- build
# Large Python DTests
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_repeat:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode_repeat:
requires:
- start_j8_dtests_large
- build
# Java 8 upgrade tests (on request)
- start_upgrade_tests:
type: approval
@ -494,6 +571,20 @@ jobs:
- log_environment
- run_parallel_junit_tests
j8_cqlshlib_tests:
<<: *j8_small_executor
steps:
- attach_workspace:
at: /home/cassandra
- run_cqlshlib_tests
j8_cqlshlib_cython_tests:
<<: *j8_small_executor
steps:
- attach_workspace:
at: /home/cassandra
- run_cqlshlib_cython_tests
j8_jvm_dtests:
<<: *j8_small_par_executor
steps:
@ -523,6 +614,7 @@ jobs:
steps:
- attach_workspace:
at: /home/cassandra
- log_environment
- run_junit_tests:
target: long-test
@ -582,6 +674,34 @@ jobs:
file_tag: j8_without_vnodes
pytest_extra_args: '--skip-resource-intensive-tests'
j8_dtests_large_vnode:
<<: *j8_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- create_dtest_containers:
file_tag: j8_large_with_vnodes
run_dtests_extra_args: '--use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests'
- run_dtests:
file_tag: j8_large_with_vnodes
pytest_extra_args: '--use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_dtests_large:
<<: *j8_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- create_dtest_containers:
file_tag: j8_large_without_vnodes
run_dtests_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
- run_dtests:
file_tag: j8_large_without_vnodes
pytest_extra_args: '--num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_upgrade_dtests:
<<: *j8_par_executor
steps:
@ -693,6 +813,36 @@ jobs:
count: ${REPEATED_DTESTS_COUNT}
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
j8_dtests_large_repeat:
<<: *j8_repeated_dtest_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- run_repeated_dtest:
tests: ${REPEATED_LARGE_DTESTS}
vnodes: "false"
upgrade: "false"
count: ${REPEATED_LARGE_DTESTS_COUNT}
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
j8_dtests_large_vnode_repeat:
<<: *j8_repeated_dtest_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- run_repeated_dtest:
tests: ${REPEATED_LARGE_DTESTS}
vnodes: "true"
upgrade: "false"
count: ${REPEATED_LARGE_DTESTS_COUNT}
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
j8_upgrade_dtests_repeat:
<<: *j8_repeated_upgrade_dtest_executor
steps:
@ -901,8 +1051,6 @@ commands:
steps:
- run:
name: Run Unit Tests (<<parameters.target>>)
# Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
# based on Java 11 builds.
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
@ -935,8 +1083,6 @@ commands:
steps:
- run:
name: Run Unit Tests (<<parameters.target>>)
# Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
# based on Java 11 builds.
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
@ -960,6 +1106,41 @@ commands:
path: /tmp/cassandra/build/test/logs
destination: logs
run_cqlshlib_tests:
parameters:
no_output_timeout:
type: string
default: 15m
steps:
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: <<parameters.no_output_timeout>>
- store_test_results:
path: /tmp/cassandra/pylib
run_cqlshlib_cython_tests:
parameters:
no_output_timeout:
type: string
default: 15m
steps:
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
export cython="yes"
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: <<parameters.no_output_timeout>>
- store_test_results:
path: /tmp/cassandra/pylib
create_venv:
steps:
- run:
@ -1048,7 +1229,12 @@ commands:
mkdir -p /tmp/results/dtests
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
if [ ! -z "$SPLIT_TESTS" ]; then
set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
else
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
(exit 1)
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1382,7 +1568,7 @@ commands:
default: ""
steps:
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "<<parameters.tests>>" == "<nil>" ]; then
@ -1425,7 +1611,7 @@ commands:
vnodes_args=""
if <<parameters.vnodes>>; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""

27
.circleci/config-2_1.yml.high_res.patch
View File

@ -1,5 +1,5 @@
--- config-2_1.yml 2022-11-05 18:28:58.000000000 -0400
+++ config-2_1.yml.HIGHRES 2022-11-05 18:47:13.000000000 -0400
--- config-2_1.yml 2023-02-07 20:47:06.000000000 -0500
+++ config-2_1.yml.HIGHRES 2023-02-07 20:49:38.000000000 -0500
@@ -40,8 +40,8 @@
CASSANDRA_SKIP_SYNC: true
DTEST_REPO: https://github.com/apache/cassandra-dtest.git
@ -11,7 +11,7 @@
# Whether the repeated test iterations should stop on the first failure by default.
REPEATED_TESTS_STOP_ON_FAILURE: false
@@ -125,46 +125,50 @@
@@ -134,14 +134,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
@ -28,6 +28,9 @@
+ exec_resource_class: xlarge
+ parallelism: 2
j8_small_executor: &j8_small_executor
executor:
@@ -152,34 +152,38 @@
j8_medium_par_executor: &j8_medium_par_executor
executor:
name: java8-executor
@ -73,3 +76,21 @@
separate_jobs: &separate_jobs
jobs:
@@ -675,7 +679,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j8_dtests_large_vnode:
- <<: *j8_par_executor
+ <<: *j8_seq_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -689,7 +693,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_dtests_large:
- <<: *j8_par_executor
+ <<: *j8_seq_executor
steps:
- attach_workspace:
at: /home/cassandra

44
.circleci/config-2_1.yml.mid_res.patch
View File

@ -1,9 +1,10 @@
--- config-2_1.yml 2022-12-23 09:43:57.000000000 -0500
+++ config-2_1.yml.MIDRES 2022-12-23 09:45:18.000000000 -0500
@@ -126,45 +126,65 @@
--- config-2_1.yml 2023-02-02 21:03:34.000000000 -0500
+++ config-2_1.yml.MIDRES 2023-02-02 21:04:03.000000000 -0500
@@ -134,14 +134,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
#exec_resource_class: xlarge
- #exec_resource_class: xlarge
- parallelism: 4
+ exec_resource_class: medium
+ parallelism: 25
@ -11,17 +12,20 @@
j8_small_par_executor: &j8_small_par_executor
executor:
name: java8-executor
#exec_resource_class: xlarge
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: large
+ parallelism: 10
j8_small_executor: &j8_small_executor
executor:
@@ -152,34 +152,51 @@
j8_medium_par_executor: &j8_medium_par_executor
executor:
name: java8-executor
#exec_resource_class: xlarge
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: large
+ exec_resource_class: xlarge
+ parallelism: 4
+
+j8_large_par_executor: &j8_large_par_executor
@ -73,7 +77,7 @@
separate_jobs: &separate_jobs
jobs:
@@ -555,7 +575,7 @@
@@ -647,7 +664,7 @@
target: stress-test
j8_dtests_vnode:
@ -82,7 +86,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -569,7 +589,7 @@
@@ -661,7 +678,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
j8_dtests:
@ -91,16 +95,34 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -583,7 +603,7 @@
@@ -675,7 +692,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j8_dtests_large_vnode:
- <<: *j8_par_executor
+ <<: *j8_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -689,7 +706,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_dtests_large:
- <<: *j8_par_executor
+ <<: *j8_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -703,7 +720,7 @@
pytest_extra_args: '--num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_upgrade_dtests:
- <<: *j8_par_executor
+ <<: *j8_very_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -708,7 +728,7 @@
@@ -858,7 +875,7 @@
count: ${REPEATED_UPGRADE_DTESTS_COUNT}
j8_dtests_offheap:

665
.circleci/config.yml
View File

@ -121,6 +121,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -304,6 +306,87 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_with_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_with_vnodes_raw /tmp/all_dtest_tests_j8_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_with_vnodes_raw > /tmp/all_dtest_tests_j8_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_with_vnodes > /tmp/split_dtest_tests_j8_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -487,6 +570,67 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_cython_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
export cython="yes"
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -521,7 +665,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -564,7 +708,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -612,6 +756,135 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -795,6 +1068,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -978,6 +1253,87 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_without_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_without_vnodes_raw /tmp/all_dtest_tests_j8_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_without_vnodes_raw > /tmp/all_dtest_tests_j8_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_without_vnodes > /tmp/split_dtest_tests_j8_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_without_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1012,7 +1368,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
@ -1055,7 +1411,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1103,6 +1459,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1203,6 +1561,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1237,7 +1597,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -1280,7 +1640,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1328,6 +1688,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1368,7 +1730,7 @@ jobs:
- run:
name: Run dtests (j8_upgradetests_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1405,6 +1767,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1588,6 +1952,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1699,6 +2065,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1764,6 +2132,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1826,7 +2196,7 @@ jobs:
- run:
name: Run dtests (j8_dtests_offheap)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1863,6 +2233,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1974,6 +2346,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2014,7 +2388,7 @@ jobs:
- run:
name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -2051,6 +2425,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2091,7 +2467,7 @@ jobs:
- run:
name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -2128,6 +2504,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2311,6 +2689,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2489,6 +2869,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2523,7 +2905,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -2566,7 +2948,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -2614,6 +2996,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2797,6 +3181,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2908,6 +3294,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2926,6 +3314,28 @@ jobs:
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (long-test)
command: |
@ -2973,6 +3383,193 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3084,6 +3681,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3181,6 +3780,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3210,6 +3811,18 @@ workflows:
requires:
- start_j8_jvm_dtests
- build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
requires:
- start_j8_cqlshlib_tests
- build
- start_j8_cqlshlib_cython_tests:
type: approval
- j8_cqlshlib_cython_tests:
requires:
- start_j8_cqlshlib_cython_tests
- build
- start_utests_long:
type: approval
- utests_long:
@ -3258,6 +3871,18 @@ workflows:
requires:
- start_j8_dtests_vnode
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- start_j8_dtests_large_vnode:
type: approval
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large_vnode
- build
- start_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
@ -3283,6 +3908,12 @@ workflows:
- j8_jvm_dtests:
requires:
- build
- j8_cqlshlib_tests:
requires:
- build
- j8_cqlshlib_cython_tests:
requires:
- build
- start_utests_long:
type: approval
- utests_long:
@ -3322,6 +3953,16 @@ workflows:
- j8_dtests_vnode:
requires:
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large
- build
- start_upgrade_tests:
type: approval
- j8_upgrade_dtests:

665
.circleci/config.yml.HIGHRES
View File

@ -121,6 +121,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -304,6 +306,87 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_with_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_with_vnodes_raw /tmp/all_dtest_tests_j8_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_with_vnodes_raw > /tmp/all_dtest_tests_j8_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_with_vnodes > /tmp/split_dtest_tests_j8_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -487,6 +570,67 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_cython_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
export cython="yes"
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -521,7 +665,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -564,7 +708,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -612,6 +756,135 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 100
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -795,6 +1068,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -978,6 +1253,87 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_without_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_without_vnodes_raw /tmp/all_dtest_tests_j8_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_without_vnodes_raw > /tmp/all_dtest_tests_j8_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_without_vnodes > /tmp/split_dtest_tests_j8_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_without_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1012,7 +1368,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
@ -1055,7 +1411,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1103,6 +1459,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1203,6 +1561,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1237,7 +1597,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -1280,7 +1640,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1328,6 +1688,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1368,7 +1730,7 @@ jobs:
- run:
name: Run dtests (j8_upgradetests_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1405,6 +1767,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1588,6 +1952,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1699,6 +2065,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1764,6 +2132,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1826,7 +2196,7 @@ jobs:
- run:
name: Run dtests (j8_dtests_offheap)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1863,6 +2233,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1974,6 +2346,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2014,7 +2388,7 @@ jobs:
- run:
name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -2051,6 +2425,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2091,7 +2467,7 @@ jobs:
- run:
name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -2128,6 +2504,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2311,6 +2689,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2489,6 +2869,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2523,7 +2905,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -2566,7 +2948,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -2614,6 +2996,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2797,6 +3181,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2908,6 +3294,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2926,6 +3314,28 @@ jobs:
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (long-test)
command: |
@ -2973,6 +3383,193 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 100
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3084,6 +3681,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3181,6 +3780,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3210,6 +3811,18 @@ workflows:
requires:
- start_j8_jvm_dtests
- build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
requires:
- start_j8_cqlshlib_tests
- build
- start_j8_cqlshlib_cython_tests:
type: approval
- j8_cqlshlib_cython_tests:
requires:
- start_j8_cqlshlib_cython_tests
- build
- start_utests_long:
type: approval
- utests_long:
@ -3258,6 +3871,18 @@ workflows:
requires:
- start_j8_dtests_vnode
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- start_j8_dtests_large_vnode:
type: approval
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large_vnode
- build
- start_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
@ -3283,6 +3908,12 @@ workflows:
- j8_jvm_dtests:
requires:
- build
- j8_cqlshlib_tests:
requires:
- build
- j8_cqlshlib_cython_tests:
requires:
- build
- start_utests_long:
type: approval
- utests_long:
@ -3322,6 +3953,16 @@ workflows:
- j8_dtests_vnode:
requires:
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large
- build
- start_upgrade_tests:
type: approval
- j8_upgrade_dtests:

665
.circleci/config.yml.LOWRES
View File

@ -121,6 +121,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -304,6 +306,87 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_with_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_with_vnodes_raw /tmp/all_dtest_tests_j8_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_with_vnodes_raw > /tmp/all_dtest_tests_j8_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_with_vnodes > /tmp/split_dtest_tests_j8_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -487,6 +570,67 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_cython_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
export cython="yes"
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -521,7 +665,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -564,7 +708,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -612,6 +756,135 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -795,6 +1068,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -978,6 +1253,87 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_without_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_without_vnodes_raw /tmp/all_dtest_tests_j8_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_without_vnodes_raw > /tmp/all_dtest_tests_j8_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_without_vnodes > /tmp/split_dtest_tests_j8_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_without_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1012,7 +1368,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
@ -1055,7 +1411,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1103,6 +1459,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1203,6 +1561,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1237,7 +1597,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -1280,7 +1640,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1328,6 +1688,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1368,7 +1730,7 @@ jobs:
- run:
name: Run dtests (j8_upgradetests_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1405,6 +1767,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1588,6 +1952,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1699,6 +2065,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1764,6 +2132,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1826,7 +2196,7 @@ jobs:
- run:
name: Run dtests (j8_dtests_offheap)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1863,6 +2233,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1974,6 +2346,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2014,7 +2388,7 @@ jobs:
- run:
name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -2051,6 +2425,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2091,7 +2467,7 @@ jobs:
- run:
name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -2128,6 +2504,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2311,6 +2689,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2489,6 +2869,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2523,7 +2905,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -2566,7 +2948,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -2614,6 +2996,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2797,6 +3181,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2908,6 +3294,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2926,6 +3314,28 @@ jobs:
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (long-test)
command: |
@ -2973,6 +3383,193 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3084,6 +3681,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3181,6 +3780,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3210,6 +3811,18 @@ workflows:
requires:
- start_j8_jvm_dtests
- build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
requires:
- start_j8_cqlshlib_tests
- build
- start_j8_cqlshlib_cython_tests:
type: approval
- j8_cqlshlib_cython_tests:
requires:
- start_j8_cqlshlib_cython_tests
- build
- start_utests_long:
type: approval
- utests_long:
@ -3258,6 +3871,18 @@ workflows:
requires:
- start_j8_dtests_vnode
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- start_j8_dtests_large_vnode:
type: approval
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large_vnode
- build
- start_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
@ -3283,6 +3908,12 @@ workflows:
- j8_jvm_dtests:
requires:
- build
- j8_cqlshlib_tests:
requires:
- build
- j8_cqlshlib_cython_tests:
requires:
- build
- start_utests_long:
type: approval
- utests_long:
@ -3322,6 +3953,16 @@ workflows:
- j8_dtests_vnode:
requires:
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large
- build
- start_upgrade_tests:
type: approval
- j8_upgrade_dtests:

667
.circleci/config.yml.MIDRES
View File

@ -21,7 +21,7 @@ jobs:
j8_jvm_upgrade_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: large
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
@ -121,6 +121,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -304,6 +306,87 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_with_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_with_vnodes_raw /tmp/all_dtest_tests_j8_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_with_vnodes_raw > /tmp/all_dtest_tests_j8_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_with_vnodes > /tmp/split_dtest_tests_j8_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -487,6 +570,67 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_cython_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
export cython="yes"
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -521,7 +665,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -564,7 +708,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -612,6 +756,135 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: large
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 25
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -795,6 +1068,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -978,6 +1253,87 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_without_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_without_vnodes_raw /tmp/all_dtest_tests_j8_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_without_vnodes_raw > /tmp/all_dtest_tests_j8_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_without_vnodes > /tmp/split_dtest_tests_j8_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_without_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1012,7 +1368,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
@ -1055,7 +1411,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1103,6 +1459,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1203,6 +1561,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1237,7 +1597,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -1280,7 +1640,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1328,6 +1688,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1368,7 +1730,7 @@ jobs:
- run:
name: Run dtests (j8_upgradetests_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1405,6 +1767,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1588,6 +1952,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1699,6 +2065,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1764,6 +2132,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1826,7 +2196,7 @@ jobs:
- run:
name: Run dtests (j8_dtests_offheap)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1863,6 +2233,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1974,6 +2346,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2014,7 +2388,7 @@ jobs:
- run:
name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -2051,6 +2425,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2091,7 +2467,7 @@ jobs:
- run:
name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -2128,6 +2504,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2311,6 +2689,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2489,6 +2869,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2523,7 +2905,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -2566,7 +2948,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -2614,6 +2996,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2797,6 +3181,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2908,6 +3294,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2926,6 +3314,28 @@ jobs:
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (long-test)
command: |
@ -2973,6 +3383,193 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: large
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 25
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3084,6 +3681,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3181,6 +3780,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -3210,6 +3811,18 @@ workflows:
requires:
- start_j8_jvm_dtests
- build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
requires:
- start_j8_cqlshlib_tests
- build
- start_j8_cqlshlib_cython_tests:
type: approval
- j8_cqlshlib_cython_tests:
requires:
- start_j8_cqlshlib_cython_tests
- build
- start_utests_long:
type: approval
- utests_long:
@ -3258,6 +3871,18 @@ workflows:
requires:
- start_j8_dtests_vnode
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- start_j8_dtests_large_vnode:
type: approval
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large_vnode
- build
- start_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
@ -3283,6 +3908,12 @@ workflows:
- j8_jvm_dtests:
requires:
- build
- j8_cqlshlib_tests:
requires:
- build
- j8_cqlshlib_cython_tests:
requires:
- build
- start_utests_long:
type: approval
- utests_long:
@ -3322,6 +3953,16 @@ workflows:
- j8_dtests_vnode:
requires:
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large
- build
- start_upgrade_tests:
type: approval
- j8_upgrade_dtests:

8
.circleci/generate.sh
View File

@ -53,6 +53,8 @@ print_help()
echo " -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500"
echo " -e REPEATED_DTESTS=cdc_test.py cqlsh_tests/test_cqlsh.py::TestCqlshSmoke"
echo " -e REPEATED_DTESTS_COUNT=500"
echo " -e REPEATED_LARGE_DTESTS=replace_address_test.py::TestReplaceAddress::test_replace_stopped_node"
echo " -e REPEATED_LARGE_DTESTS=100"
echo " -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py upgrade_tests/paging_test.py"
echo " -e REPEATED_UPGRADE_DTESTS_COUNT=25"
echo " -e REPEATED_ANT_TEST_TARGET=testsome"
@ -120,6 +122,8 @@ if $has_env_vars && $check_env_vars; then
[ "$key" != "REPEATED_JVM_UPGRADE_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_DTESTS" ] &&
[ "$key" != "REPEATED_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_LARGE_DTESTS" ] &&
[ "$key" != "REPEATED_LARGE_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_UPGRADE_DTESTS" ] &&
[ "$key" != "REPEATED_UPGRADE_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_ANT_TEST_TARGET" ] &&
@ -281,6 +285,10 @@ delete_repeated_jobs()
delete_job "$1" "j11_dtests_repeat"
delete_job "$1" "j11_dtests_vnode_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_LARGE_DTESTS=")); then
delete_job "$1" "j8_dtests_large_repeat"
delete_job "$1" "j8_dtests_large_vnode_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_UPGRADE_DTESTS=")); then
delete_job "$1" "j8_upgrade_dtests_repeat"
fi

8
.circleci/readme.md
View File

@ -77,6 +77,7 @@ generate.sh -m \
-e REPEATED_JVM_DTESTS_COUNT=500 \
-e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500 \
-e REPEATED_DTESTS_COUNT=500 \
-e REPEATED_LARGE_DTESTS_COUNT=100 \
-e REPEATED_UPGRADE_DTESTS_COUNT=25 \
-e REPEATED_ANT_TEST_COUNT=500
```
@ -92,6 +93,7 @@ generate.sh -m \
-e REPEATED_UTESTS_LONG=org.apache.cassandra.io.sstable.CQLSSTableWriterLongTest#testWideRow \
-e REPEATED_UTESTS_STRESS=org.apache.cassandra.stress.generate.DistributionGaussianTest \
-e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \
-e REPEATED_LARGE_DTESTS=replace_address_test.py::TestReplaceAddress::test_replace_stopped_node \
-e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest#testPaging \
-e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py \
-e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest
@ -117,6 +119,8 @@ generate.sh -m \
-e REPEATED_UTESTS_STRESS_COUNT=500 \
-e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \
-e REPEATED_DTESTS_COUNT=500 \
-e REPEATED_LARGE_DTESTS=replace_address_test.py,materialized_views_test.py \
-e REPEATED_LARGE_DTESTS_COUNT=100 \
-e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest#testPaging \
-e REPEATED_JVM_DTESTS_COUNT=500 \
-e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py \
@ -129,6 +133,10 @@ generate.sh -m \
-e REPEATED_ANT_TEST_COUNT=500
```
WARNING: The Python large DTests test_network_topology_strategy and test_network_topology_strategy_each_quorum require
XLarge containers in order to be run in a loop. Otherwise, you would see NO HOST AVAILABLE if you run them with the
default medium config and Large containers.
## Updating the config
For configuration changes meant to be permanent in the Apache repo you should never edit
the `config.yml` file manually. Instead, you should edit the `config-2_1.yml` file and then