natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add Dtest-large and cqlshlib tests to CircleCI; Fix num tokens in CircleCI tests to match the default value; Fix CircleCI Python tests split bug; Fix CircleCI long unit tests 

patch by Ekaterina Dimitrova; reviewed by Andres de la Pena for CASSANDRA-18001
This commit is contained in:
Ekaterina Dimitrova 2022-10-31 18:10:05 -04:00
parent fa18ea1852
commit 476d31bce2
9 changed files with 2808 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

213
.circleci/config-2_1.yml
View File

@ -88,6 +88,15 @@ default_env_vars: &default_env_vars
# The number of times that the manually specified Python dtests should be run.
REPEATED_DTESTS_COUNT: 500
# Comma-separated list of Python large dtests that should be repeatedly run, for example:
# REPEATED_LARGE_DTESTS: replace_address_test.py
# REPEATED_LARGE_DTESTS: replace_address_test.py::TestReplaceAddress
# REPEATED_LARGE_DTESTS: replace_address_test.py::TestReplaceAddress::test_replace_stopped_node
# REPEATED_LARGE_DTESTS: replace_address_test.py,materialized_views_test.py
REPEATED_LARGE_DTESTS:
# The number of times that the manually specified Python large dtests should be run.
REPEATED_LARGE_DTESTS_COUNT: 100
# Comma-separated list of Python dtests that should be repeatedly run, for example:
# REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py
# REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py::TestCQLNodes2RF1_Upgrade_current_4_0_x_To_indev_4_0_x
@ -127,6 +136,12 @@ j8_small_par_executor: &j8_small_par_executor
#exec_resource_class: xlarge
parallelism: 1
j8_small_executor: &j8_small_executor
executor:
name: java8-executor
exec_resource_class: medium
parallelism: 1
j8_medium_par_executor: &j8_medium_par_executor
executor:
name: java8-executor
@ -191,6 +206,18 @@ separate_jobs: &separate_jobs
requires:
- start_j8_jvm_dtests_repeat
- build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
requires:
- start_j8_cqlshlib_tests
- build
- start_j8_cqlshlib_cython_tests:
type: approval
- j8_cqlshlib_cython_tests:
requires:
- start_j8_cqlshlib_cython_tests
- build
# specialized unit tests (all run using Java 8)
- start_utests_long:
type: approval
@ -234,7 +261,7 @@ separate_jobs: &separate_jobs
requires:
- start_jvm_upgrade_dtests_repeat
- j8_dtest_jars_build
# Java 8 dtests
# Python DTests
- start_j8_dtests:
type: approval
- j8_dtests:
@ -259,7 +286,32 @@ separate_jobs: &separate_jobs
requires:
- start_j8_dtests_vnode_repeat
- build
# Java 8 upgrade tests
# Python large DTests
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- start_j8_dtests_large_repeat:
type: approval
- j8_dtests_large_repeat:
requires:
- start_j8_dtests_large_repeat
- build
- start_j8_dtests_large_vnode:
type: approval
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large_vnode
- build
- start_j8_dtests_large_vnode_repeat:
type: approval
- j8_dtests_large_vnode_repeat:
requires:
- start_j8_dtests_large_vnode_repeat
- build
# Python upgrade tests
- start_j8_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
@ -301,6 +353,12 @@ pre-commit_jobs: &pre-commit_jobs
- j8_jvm_dtests_repeat:
requires:
- build
- j8_cqlshlib_tests:
requires:
- build
- j8_cqlshlib_cython_tests:
requires:
- build
# specialized unit tests (all run on request using Java 8)
- start_utests_long:
type: approval
@ -334,7 +392,7 @@ pre-commit_jobs: &pre-commit_jobs
- j8_jvm_upgrade_dtests_repeat:
requires:
- j8_dtest_jars_build
# Java 8 dtests
# Python DTests
- j8_dtests:
requires:
- build
@ -347,6 +405,25 @@ pre-commit_jobs: &pre-commit_jobs
- j8_dtests_vnode_repeat:
requires:
- build
# Large Python DTests
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_repeat:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode_repeat:
requires:
- start_j8_dtests_large
- build
# Java 8 upgrade tests (on request)
- start_upgrade_dtests:
type: approval
@ -416,6 +493,20 @@ jobs:
- log_environment
- run_parallel_junit_tests
j8_cqlshlib_tests:
<<: *j8_small_executor
steps:
- attach_workspace:
at: /home/cassandra
- run_cqlshlib_tests
j8_cqlshlib_cython_tests:
<<: *j8_small_executor
steps:
- attach_workspace:
at: /home/cassandra
- run_cqlshlib_cython_tests
j8_jvm_dtests:
<<: *j8_seq_executor
steps:
@ -445,6 +536,7 @@ jobs:
steps:
- attach_workspace:
at: /home/cassandra
- log_environment
- run_junit_tests:
target: long-test
@ -486,6 +578,34 @@ jobs:
file_tag: j8_without_vnodes
pytest_extra_args: '--skip-resource-intensive-tests'
j8_dtests_large_vnode:
<<: *j8_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- create_dtest_containers:
file_tag: j8_large_with_vnodes
run_dtests_extra_args: '--use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests'
- run_dtests:
file_tag: j8_large_with_vnodes
pytest_extra_args: '--use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_dtests_large:
<<: *j8_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- create_dtest_containers:
file_tag: j8_large_without_vnodes
run_dtests_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
- run_dtests:
file_tag: j8_large_without_vnodes
pytest_extra_args: '--num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_upgrade_dtests:
<<: *j8_par_executor
steps:
@ -589,6 +709,36 @@ jobs:
count: ${REPEATED_DTESTS_COUNT}
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
j8_dtests_large_repeat:
<<: *j8_repeated_dtest_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- run_repeated_dtest:
tests: ${REPEATED_LARGE_DTESTS}
vnodes: "false"
upgrade: "false"
count: ${REPEATED_LARGE_DTESTS_COUNT}
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
j8_dtests_large_vnode_repeat:
<<: *j8_repeated_dtest_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- run_repeated_dtest:
tests: ${REPEATED_LARGE_DTESTS}
vnodes: "true"
upgrade: "false"
count: ${REPEATED_LARGE_DTESTS_COUNT}
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests"
j8_upgrade_dtests_repeat:
<<: *j8_repeated_upgrade_dtest_executor
steps:
@ -764,8 +914,6 @@ commands:
steps:
- run:
name: Run Unit Tests (<<parameters.target>>)
# Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
# based on Java 11 builds.
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
@ -773,7 +921,7 @@ commands:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
ant clean <<parameters.target>> -Dno-build-test=true
ant <<parameters.target>> -Dno-build-test=true
no_output_timeout: <<parameters.no_output_timeout>>
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -798,8 +946,6 @@ commands:
steps:
- run:
name: Run Unit Tests (<<parameters.target>>)
# Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
# based on Java 11 builds.
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
@ -823,6 +969,41 @@ commands:
path: /tmp/cassandra/build/test/logs
destination: logs
run_cqlshlib_tests:
parameters:
no_output_timeout:
type: string
default: 15m
steps:
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: <<parameters.no_output_timeout>>
- store_test_results:
path: /tmp/cassandra/pylib
run_cqlshlib_cython_tests:
parameters:
no_output_timeout:
type: string
default: 15m
steps:
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
export cython="yes"
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: <<parameters.no_output_timeout>>
- store_test_results:
path: /tmp/cassandra/pylib
create_venv:
steps:
- run:
@ -911,7 +1092,12 @@ commands:
mkdir -p /tmp/results/dtests
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
if [ ! -z "$SPLIT_TESTS" ]; then
set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level="DEBUG" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
else
echo "Tune your parallelism, there are more containers than test classes. Nothing to do in this container"
(exit 1)
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1230,9 +1416,12 @@ commands:
type: string
stop_on_failure:
type: string
extra_dtest_args:
type: string
default: ""
steps:
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "<<parameters.tests>>" == "<nil>" ]; then
@ -1275,7 +1464,7 @@ commands:
vnodes_args=""
if <<parameters.vnodes>>; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1284,7 +1473,7 @@ commands:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir <<parameters.extra_dtest_args>> $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:

36
.circleci/config-2_1.yml.high_res.patch
View File

@ -1,5 +1,5 @@
--- config-2_1.yml 2022-11-03 12:41:29.000000000 -0400
+++ config-2_1.yml.HIGHRES 2022-11-03 12:48:44.000000000 -0400
--- config-2_1.yml 2023-02-07 19:05:10.000000000 -0500
+++ config-2_1.yml.HIGHRES 2023-02-07 19:25:03.000000000 -0500
@@ -40,8 +40,8 @@
CASSANDRA_SKIP_SYNC: true
DTEST_REPO: https://github.com/apache/cassandra-dtest.git
@ -11,7 +11,7 @@
# Whether the repeated test iterations should stop on the first failure by default.
REPEATED_TESTS_STOP_ON_FAILURE: false
@@ -118,46 +118,50 @@
@@ -127,14 +127,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
@ -28,6 +28,9 @@
+ exec_resource_class: xlarge
+ parallelism: 2
j8_small_executor: &j8_small_executor
executor:
@@ -145,34 +145,38 @@
j8_medium_par_executor: &j8_medium_par_executor
executor:
name: java8-executor
@ -73,3 +76,30 @@
separate_jobs: &separate_jobs
jobs:
@@ -579,7 +583,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j8_dtests_large_vnode:
- <<: *j8_par_executor
+ <<: *j8_seq_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -593,7 +597,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_dtests_large:
- <<: *j8_par_executor
+ <<: *j8_seq_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1187,7 +1191,7 @@
# Put manually specified tests and automatically detected tests together, removing duplicates
tests=$(echo <<parameters.tests>> | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
echo "Tests to be repeated: ${tests}"
-
+
# Prepare the testtag for the target, used by the test macro in build.xml to group the output files
target=<<parameters.target>>
testtag=""

46
.circleci/config-2_1.yml.mid_res.patch
View File

@ -1,6 +1,6 @@
--- config-2_1.yml 2022-12-22 17:06:24.000000000 -0500
+++ config-2_1.yml.MIDRES 2022-12-22 17:20:36.000000000 -0500
@@ -119,45 +119,65 @@
--- config-2_1.yml 2023-02-07 19:05:10.000000000 -0500
+++ config-2_1.yml.MIDRES 2023-02-07 19:17:39.000000000 -0500
@@ -128,13 +128,15 @@
executor:
name: java8-executor
#exec_resource_class: xlarge
@ -16,12 +16,15 @@
+ exec_resource_class: large
+ parallelism: 10
j8_small_executor: &j8_small_executor
executor:
@@ -145,34 +147,51 @@
j8_medium_par_executor: &j8_medium_par_executor
executor:
name: java8-executor
#exec_resource_class: xlarge
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: large
+ exec_resource_class: xlarge
+ parallelism: 4
+
+j8_large_par_executor: &j8_large_par_executor
@ -73,7 +76,7 @@
separate_jobs: &separate_jobs
jobs:
@@ -459,7 +479,7 @@
@@ -551,7 +570,7 @@
target: testclasslist-compression
j8_dtests_vnode:
@ -82,7 +85,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -473,7 +493,7 @@
@@ -565,7 +584,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
j8_dtests:
@ -91,12 +94,39 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -487,7 +507,7 @@
@@ -579,7 +598,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j8_dtests_large_vnode:
- <<: *j8_par_executor
+ <<: *j8_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -593,7 +612,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_dtests_large:
- <<: *j8_par_executor
+ <<: *j8_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -607,7 +626,7 @@
pytest_extra_args: '--num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_upgrade_dtests:
- <<: *j8_par_executor
+ <<: *j8_very_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1187,7 +1206,7 @@
# Put manually specified tests and automatically detected tests together, removing duplicates
tests=$(echo <<parameters.tests>> | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
echo "Tests to be repeated: ${tests}"
-
+
# Prepare the testtag for the target, used by the test macro in build.xml to group the output files
target=<<parameters.target>>
testtag=""

643
.circleci/config.yml
View File

@ -119,6 +119,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -298,6 +300,267 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_with_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_with_vnodes_raw /tmp/all_dtest_tests_j8_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_with_vnodes_raw > /tmp/all_dtest_tests_j8_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_with_vnodes > /tmp/split_dtest_tests_j8_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_cython_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
export cython="yes"
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -477,6 +740,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -656,6 +921,85 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_without_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_without_vnodes_raw /tmp/all_dtest_tests_j8_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_without_vnodes_raw > /tmp/all_dtest_tests_j8_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_without_vnodes > /tmp/split_dtest_tests_j8_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_without_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -690,7 +1034,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
@ -733,7 +1077,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -742,7 +1086,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -779,6 +1123,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -877,6 +1223,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -911,7 +1259,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -954,7 +1302,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -963,7 +1311,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1000,6 +1348,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1040,7 +1390,7 @@ jobs:
- run:
name: Run dtests (j8_upgradetests_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1075,6 +1425,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1254,6 +1606,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1363,6 +1717,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1403,7 +1759,7 @@ jobs:
- run:
name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1438,6 +1794,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1478,7 +1836,7 @@ jobs:
- run:
name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1513,6 +1871,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1692,6 +2052,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1868,6 +2230,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1902,7 +2266,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -1945,7 +2309,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1954,7 +2318,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1991,6 +2355,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2100,6 +2466,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2118,6 +2486,28 @@ jobs:
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (long-test)
command: |
@ -2127,7 +2517,7 @@ jobs:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
ant clean long-test -Dno-build-test=true
ant long-test -Dno-build-test=true
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2163,6 +2553,189 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2272,6 +2845,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2367,6 +2942,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2396,6 +2973,18 @@ workflows:
requires:
- start_j8_jvm_dtests
- build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
requires:
- start_j8_cqlshlib_tests
- build
- start_j8_cqlshlib_cython_tests:
type: approval
- j8_cqlshlib_cython_tests:
requires:
- start_j8_cqlshlib_cython_tests
- build
- start_utests_long:
type: approval
- utests_long:
@ -2432,6 +3021,18 @@ workflows:
requires:
- start_j8_dtests_vnode
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- start_j8_dtests_large_vnode:
type: approval
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large_vnode
- build
- start_j8_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
@ -2451,6 +3052,12 @@ workflows:
- j8_jvm_dtests:
requires:
- build
- j8_cqlshlib_tests:
requires:
- build
- j8_cqlshlib_cython_tests:
requires:
- build
- start_utests_long:
type: approval
- utests_long:
@ -2478,6 +3085,16 @@ workflows:
- j8_dtests_vnode:
requires:
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large
- build
- start_upgrade_dtests:
type: approval
- j8_upgrade_dtests:

643
.circleci/config.yml.HIGHRES
View File

@ -119,6 +119,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -298,6 +300,267 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_with_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_with_vnodes_raw /tmp/all_dtest_tests_j8_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_with_vnodes_raw > /tmp/all_dtest_tests_j8_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_with_vnodes > /tmp/split_dtest_tests_j8_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_cython_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
export cython="yes"
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 100
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -477,6 +740,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -656,6 +921,85 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_without_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_without_vnodes_raw /tmp/all_dtest_tests_j8_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_without_vnodes_raw > /tmp/all_dtest_tests_j8_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_without_vnodes > /tmp/split_dtest_tests_j8_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_without_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -690,7 +1034,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
@ -733,7 +1077,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -742,7 +1086,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -779,6 +1123,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -877,6 +1223,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -911,7 +1259,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -954,7 +1302,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -963,7 +1311,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1000,6 +1348,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1040,7 +1390,7 @@ jobs:
- run:
name: Run dtests (j8_upgradetests_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1075,6 +1425,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1254,6 +1606,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1363,6 +1717,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1403,7 +1759,7 @@ jobs:
- run:
name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1438,6 +1794,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1478,7 +1836,7 @@ jobs:
- run:
name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1513,6 +1871,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1692,6 +2052,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1868,6 +2230,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1902,7 +2266,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -1945,7 +2309,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1954,7 +2318,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1991,6 +2355,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2100,6 +2466,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2118,6 +2486,28 @@ jobs:
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (long-test)
command: |
@ -2127,7 +2517,7 @@ jobs:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
ant clean long-test -Dno-build-test=true
ant long-test -Dno-build-test=true
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2163,6 +2553,189 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 100
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2272,6 +2845,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2367,6 +2942,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2396,6 +2973,18 @@ workflows:
requires:
- start_j8_jvm_dtests
- build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
requires:
- start_j8_cqlshlib_tests
- build
- start_j8_cqlshlib_cython_tests:
type: approval
- j8_cqlshlib_cython_tests:
requires:
- start_j8_cqlshlib_cython_tests
- build
- start_utests_long:
type: approval
- utests_long:
@ -2432,6 +3021,18 @@ workflows:
requires:
- start_j8_dtests_vnode
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- start_j8_dtests_large_vnode:
type: approval
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large_vnode
- build
- start_j8_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
@ -2451,6 +3052,12 @@ workflows:
- j8_jvm_dtests:
requires:
- build
- j8_cqlshlib_tests:
requires:
- build
- j8_cqlshlib_cython_tests:
requires:
- build
- start_utests_long:
type: approval
- utests_long:
@ -2478,6 +3085,16 @@ workflows:
- j8_dtests_vnode:
requires:
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large
- build
- start_upgrade_dtests:
type: approval
- j8_upgrade_dtests:

643
.circleci/config.yml.LOWRES
View File

@ -119,6 +119,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -298,6 +300,267 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_with_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_with_vnodes_raw /tmp/all_dtest_tests_j8_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_with_vnodes_raw > /tmp/all_dtest_tests_j8_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_with_vnodes > /tmp/split_dtest_tests_j8_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_cython_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
export cython="yes"
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -477,6 +740,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -656,6 +921,85 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_without_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_without_vnodes_raw /tmp/all_dtest_tests_j8_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_without_vnodes_raw > /tmp/all_dtest_tests_j8_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_without_vnodes > /tmp/split_dtest_tests_j8_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_without_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -690,7 +1034,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
@ -733,7 +1077,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -742,7 +1086,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -779,6 +1123,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -877,6 +1223,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -911,7 +1259,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -954,7 +1302,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -963,7 +1311,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1000,6 +1348,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1040,7 +1390,7 @@ jobs:
- run:
name: Run dtests (j8_upgradetests_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1075,6 +1425,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1254,6 +1606,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1363,6 +1717,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1403,7 +1759,7 @@ jobs:
- run:
name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1438,6 +1794,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1478,7 +1836,7 @@ jobs:
- run:
name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1513,6 +1871,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1692,6 +2052,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1868,6 +2230,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1902,7 +2266,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -1945,7 +2309,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1954,7 +2318,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1991,6 +2355,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2100,6 +2466,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2118,6 +2486,28 @@ jobs:
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (long-test)
command: |
@ -2127,7 +2517,7 @@ jobs:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
ant clean long-test -Dno-build-test=true
ant long-test -Dno-build-test=true
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2163,6 +2553,189 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2272,6 +2845,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2367,6 +2942,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2396,6 +2973,18 @@ workflows:
requires:
- start_j8_jvm_dtests
- build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
requires:
- start_j8_cqlshlib_tests
- build
- start_j8_cqlshlib_cython_tests:
type: approval
- j8_cqlshlib_cython_tests:
requires:
- start_j8_cqlshlib_cython_tests
- build
- start_utests_long:
type: approval
- utests_long:
@ -2432,6 +3021,18 @@ workflows:
requires:
- start_j8_dtests_vnode
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- start_j8_dtests_large_vnode:
type: approval
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large_vnode
- build
- start_j8_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
@ -2451,6 +3052,12 @@ workflows:
- j8_jvm_dtests:
requires:
- build
- j8_cqlshlib_tests:
requires:
- build
- j8_cqlshlib_cython_tests:
requires:
- build
- start_utests_long:
type: approval
- utests_long:
@ -2478,6 +3085,16 @@ workflows:
- j8_dtests_vnode:
requires:
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large
- build
- start_upgrade_dtests:
type: approval
- j8_upgrade_dtests:

643
.circleci/config.yml.MIDRES
View File

@ -119,6 +119,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -298,6 +300,267 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_with_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_with_vnodes_raw /tmp/all_dtest_tests_j8_large_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_with_vnodes_raw > /tmp/all_dtest_tests_j8_large_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_with_vnodes > /tmp/split_dtest_tests_j8_large_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_cython_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
export cython="yes"
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: large
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 25
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -477,6 +740,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -656,6 +921,85 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_large_without_vnodes)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_large_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --only-resource-intensive-tests --force-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_large_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_large_without_vnodes_raw /tmp/all_dtest_tests_j8_large_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_large_without_vnodes_raw > /tmp/all_dtest_tests_j8_large_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_large_without_vnodes > /tmp/split_dtest_tests_j8_large_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n"
- run:
name: Run dtests (j8_large_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_large_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --num-tokens=32 --only-resource-intensive-tests --force-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_large_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_large_without_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_large_without_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -690,7 +1034,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
@ -733,7 +1077,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -742,7 +1086,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -779,6 +1123,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -877,6 +1223,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -911,7 +1259,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -954,7 +1302,7 @@ jobs:
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -963,7 +1311,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1000,6 +1348,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1040,7 +1390,7 @@ jobs:
- run:
name: Run dtests (j8_upgradetests_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1075,6 +1425,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1254,6 +1606,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1363,6 +1717,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1403,7 +1759,7 @@ jobs:
- run:
name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1438,6 +1794,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1478,7 +1836,7 @@ jobs:
- run:
name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
@ -1513,6 +1871,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1692,6 +2052,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1868,6 +2230,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -1902,7 +2266,7 @@ jobs:
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
@ -1945,7 +2309,7 @@ jobs:
vnodes_args=""
if false; then
vnodes_args="--use-vnodes --num-tokens=16"
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
@ -1954,7 +2318,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1991,6 +2355,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2100,6 +2466,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2118,6 +2486,28 @@ jobs:
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (long-test)
command: |
@ -2127,7 +2517,7 @@ jobs:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
ant clean long-test -Dno-build-test=true
ant long-test -Dno-build-test=true
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2163,6 +2553,189 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlshlib_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Run cqlshlib Unit Tests
command: |
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra/
./pylib/cassandra-cqlsh-tests.sh $(pwd)
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/pylib
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_large_vnode_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: large
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 25
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python DTests
no_output_timeout: 15m
command: |
if [ "${REPEATED_LARGE_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_LARGE_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_LARGE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_LARGE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_LARGE_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_LARGE_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=32"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --only-resource-intensive-tests --force-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2272,6 +2845,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2367,6 +2942,8 @@ jobs:
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_LARGE_DTESTS: null
- REPEATED_LARGE_DTESTS_COUNT: 100
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
@ -2396,6 +2973,18 @@ workflows:
requires:
- start_j8_jvm_dtests
- build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
requires:
- start_j8_cqlshlib_tests
- build
- start_j8_cqlshlib_cython_tests:
type: approval
- j8_cqlshlib_cython_tests:
requires:
- start_j8_cqlshlib_cython_tests
- build
- start_utests_long:
type: approval
- utests_long:
@ -2432,6 +3021,18 @@ workflows:
requires:
- start_j8_dtests_vnode
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- start_j8_dtests_large_vnode:
type: approval
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large_vnode
- build
- start_j8_upgrade_dtests:
type: approval
- j8_upgrade_dtests:
@ -2451,6 +3052,12 @@ workflows:
- j8_jvm_dtests:
requires:
- build
- j8_cqlshlib_tests:
requires:
- build
- j8_cqlshlib_cython_tests:
requires:
- build
- start_utests_long:
type: approval
- utests_long:
@ -2478,6 +3085,16 @@ workflows:
- j8_dtests_vnode:
requires:
- build
- start_j8_dtests_large:
type: approval
- j8_dtests_large:
requires:
- start_j8_dtests_large
- build
- j8_dtests_large_vnode:
requires:
- start_j8_dtests_large
- build
- start_upgrade_dtests:
type: approval
- j8_upgrade_dtests:

8
.circleci/generate.sh
View File

@ -51,6 +51,8 @@ print_help()
echo " -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500"
echo " -e REPEATED_DTESTS=cdc_test.py cqlsh_tests/test_cqlsh.py::TestCqlshSmoke"
echo " -e REPEATED_DTESTS_COUNT=500"
echo " -e REPEATED_LARGE_DTESTS=replace_address_test.py::TestReplaceAddress::test_replace_stopped_node"
echo " -e REPEATED_LARGE_DTESTS=100"
echo " -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py upgrade_tests/paging_test.py"
echo " -e REPEATED_UPGRADE_DTESTS_COUNT=25"
echo " -e REPEATED_ANT_TEST_TARGET=testsome"
@ -116,6 +118,8 @@ if $has_env_vars && $check_env_vars; then
[ "$key" != "REPEATED_JVM_UPGRADE_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_DTESTS" ] &&
[ "$key" != "REPEATED_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_LARGE_DTESTS" ] &&
[ "$key" != "REPEATED_LARGE_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_UPGRADE_DTESTS" ] &&
[ "$key" != "REPEATED_UPGRADE_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_ANT_TEST_TARGET" ] &&
@ -271,6 +275,10 @@ delete_repeated_jobs()
delete_job "$1" "j11_dtests_repeat"
delete_job "$1" "j11_dtests_vnode_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_LARGE_DTESTS=")); then
delete_job "$1" "j8_dtests_large_repeat"
delete_job "$1" "j8_dtests_large_vnode_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_UPGRADE_DTESTS=")); then
delete_job "$1" "j8_upgrade_dtests_repeat"
fi

8
.circleci/readme.md
View File

@ -76,6 +76,7 @@ generate.sh -m \
-e REPEATED_JVM_DTESTS_COUNT=500 \
-e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500 \
-e REPEATED_DTESTS_COUNT=500 \
-e REPEATED_LARGE_DTESTS_COUNT=100 \
-e REPEATED_UPGRADE_DTESTS_COUNT=25 \
-e REPEATED_ANT_TEST_COUNT=500
```
@ -90,6 +91,7 @@ generate.sh -m \
-e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest \
-e REPEATED_UTESTS_LONG=org.apache.cassandra.io.sstable.CQLSSTableWriterLongTest#testWideRow \
-e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \
-e REPEATED_LARGE_DTESTS=replace_address_test.py::TestReplaceAddress::test_replace_stopped_node \
-e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest#testPaging \
-e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py \
-e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest
@ -113,6 +115,8 @@ generate.sh -m \
-e REPEATED_UTESTS_LONG_COUNT=100 \
-e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \
-e REPEATED_DTESTS_COUNT=500 \
-e REPEATED_LARGE_DTESTS=replace_address_test.py,materialized_views_test.py \
-e REPEATED_LARGE_DTESTS_COUNT=100 \
-e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest#testPaging \
-e REPEATED_JVM_DTESTS_COUNT=500 \
-e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py \
@ -125,6 +129,10 @@ generate.sh -m \
-e REPEATED_ANT_TEST_COUNT=500
```
WARNING: The Python large DTests test_network_topology_strategy and test_network_topology_strategy_each_quorum require
XLarge containers in order to be run in a loop. Otherwise, you would see NO HOST AVAILABLE if you run them with the
default medium config and Large containers.
## Updating the config
For configuration changes meant to be permanent in the Apache repo you should never edit
the `config.yml` file manually. Instead, you should edit the `config-2_1.yml` file and then