natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[CASSANDRA-17950] Enable dtest-offheap in CircleCI 

The dtest-offheap test was only added to the Jenkins configuration, so
this commit adds it to the CircleCI build for parity.

Patch by Derek Chen-Becker; reviewed by Ekaterina Dimitrova, Derek Chen-Becker, Michael Semb Wever, and Andres de la Pena for CASSANDRA-17950

Co-authored-by: Ekaterina Dimitrova <ekaterina.dimitrova@datastax.com>
Co-authored-by: Derek Chen-Becker <dchenbec@amazon.com>
This commit is contained in:
Derek Chen-Becker 2022-10-26 14:54:05 -06:00 committed by Ekaterina Dimitrova
parent a44209cf43
commit 7572725def
8 changed files with 1031 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
.circleci/config-2_1.yml
View File

@ -310,6 +310,19 @@ separate_jobs: &separate_jobs
requires:
- start_j8_repeated_ant_test
- build
# Java 8 off-heap dtests
- start_j8_dtests_offheap:
type: approval
- j8_dtests_offheap:
requires:
- start_j8_dtests_offheap
- build
- start_j8_dtests_offheap_repeat:
type: approval
- j8_dtests_offheap_repeat:
requires:
- start_j8_dtests_offheap_repeat
- build
pre-commit_jobs: &pre-commit_jobs
jobs:
@ -412,6 +425,17 @@ pre-commit_jobs: &pre-commit_jobs
- j8_repeated_ant_test:
requires:
- build
# Java 8 off-heap ddtests
- start_j8_dtests_offheap:
type: approval
- j8_dtests_offheap:
requires:
- start_j8_dtests_offheap
- build
- j8_dtests_offheap_repeat:
requires:
- start_j8_dtests_offheap
- build
workflows:
version: 2
@ -683,6 +707,36 @@ jobs:
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
count: ${REPEATED_UPGRADE_DTESTS_COUNT}
j8_dtests_offheap:
<<: *j8_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- log_environment
- clone_dtest
- create_venv
- create_dtest_containers:
file_tag: j8_dtests_offheap
run_dtests_extra_args: "--use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests"
- run_dtests:
file_tag: j8_dtests_offheap
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j8_dtests_offheap_repeat:
<<: *j8_repeated_dtest_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- run_repeated_dtest:
tests: ${REPEATED_DTESTS}
vnodes: "true"
upgrade: "false"
count: ${REPEATED_DTESTS_COUNT}
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
extra_dtest_args: "--use-off-heap-memtables --skip-resource-intensive-tests"
commands:
log_environment:
steps:
@ -1323,6 +1377,9 @@ commands:
type: string
stop_on_failure:
type: string
extra_dtest_args:
type: string
default: ""
steps:
- run:
name: Run repeated Python dtest
@ -1377,7 +1434,7 @@ commands:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir <<parameters.extra_dtest_args>> $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:

4
.circleci/config-2_1.yml.high_res.patch
View File

@ -1,5 +1,5 @@
--- config-2_1.yml 2022-11-03 12:52:03.000000000 -0400
+++ config-2_1.yml.HIGHRES 2022-11-03 12:53:22.000000000 -0400
--- config-2_1.yml 2022-11-05 18:28:58.000000000 -0400
+++ config-2_1.yml.HIGHRES 2022-11-05 18:47:13.000000000 -0400
@@ -40,8 +40,8 @@
CASSANDRA_SKIP_SYNC: true
DTEST_REPO: https://github.com/apache/cassandra-dtest.git

19
.circleci/config-2_1.yml.mid_res.patch
View File

@ -1,5 +1,5 @@
--- config-2_1.yml 2022-11-03 12:52:03.000000000 -0400
+++ config-2_1.yml.MIDRES 2022-11-03 12:53:22.000000000 -0400
--- config-2_1.yml 2022-11-05 18:28:58.000000000 -0400
+++ config-2_1.yml.MIDRES 2022-11-05 18:47:13.000000000 -0400
@@ -126,45 +126,65 @@
executor:
name: java8-executor
@ -73,7 +73,7 @@
separate_jobs: &separate_jobs
jobs:
@@ -531,7 +551,7 @@
@@ -555,7 +575,7 @@
target: stress-test
j8_dtests_vnode:
@ -82,7 +82,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -545,7 +565,7 @@
@@ -569,7 +589,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
j8_dtests:
@ -91,7 +91,7 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -559,7 +579,7 @@
@@ -583,7 +603,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j8_upgrade_dtests:
@ -100,3 +100,12 @@
steps:
- attach_workspace:
at: /home/cassandra
@@ -708,7 +728,7 @@
count: ${REPEATED_UPGRADE_DTESTS_COUNT}
j8_dtests_offheap:
- <<: *j8_par_executor
+ <<: *j8_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra

242
.circleci/config.yml
View File

@ -495,6 +495,131 @@ jobs:
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_offheap_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
utests_long_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -939,7 +1064,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1164,7 +1289,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1647,6 +1772,105 @@ jobs:
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_dtests_offheap)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n"
- run:
name: Run dtests (j8_dtests_offheap)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_dtests_offheap
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_dtests_offheap_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_unit_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -2351,7 +2575,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -3040,6 +3264,12 @@ workflows:
requires:
- start_upgrade_dtests
- build
- start_j8_dtests_offheap:
type: approval
- j8_dtests_offheap:
requires:
- start_j8_dtests_offheap
- build
pre-commit_tests:
jobs:
- start_pre-commit_tests:
@ -3098,3 +3328,9 @@ workflows:
requires:
- start_upgrade_tests
- build
- start_j8_dtests_offheap:
type: approval
- j8_dtests_offheap:
requires:
- start_j8_dtests_offheap
- build

242
.circleci/config.yml.HIGHRES
View File

@ -495,6 +495,131 @@ jobs:
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_offheap_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 100
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
utests_long_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -939,7 +1064,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1164,7 +1289,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1647,6 +1772,105 @@ jobs:
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 100
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_dtests_offheap)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n"
- run:
name: Run dtests (j8_dtests_offheap)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_dtests_offheap
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_dtests_offheap_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 2048M
- CCM_HEAP_NEWSIZE: 512M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_unit_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -2351,7 +2575,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -3040,6 +3264,12 @@ workflows:
requires:
- start_upgrade_dtests
- build
- start_j8_dtests_offheap:
type: approval
- j8_dtests_offheap:
requires:
- start_j8_dtests_offheap
- build
pre-commit_tests:
jobs:
- start_pre-commit_tests:
@ -3098,3 +3328,9 @@ workflows:
requires:
- start_upgrade_tests
- build
- start_j8_dtests_offheap:
type: approval
- j8_dtests_offheap:
requires:
- start_j8_dtests_offheap
- build

242
.circleci/config.yml.LOWRES
View File

@ -495,6 +495,131 @@ jobs:
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_offheap_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
utests_long_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -939,7 +1064,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1164,7 +1289,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1647,6 +1772,105 @@ jobs:
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_dtests_offheap)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n"
- run:
name: Run dtests (j8_dtests_offheap)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_dtests_offheap
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_dtests_offheap_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_unit_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -2351,7 +2575,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -3040,6 +3264,12 @@ workflows:
requires:
- start_upgrade_dtests
- build
- start_j8_dtests_offheap:
type: approval
- j8_dtests_offheap:
requires:
- start_j8_dtests_offheap
- build
pre-commit_tests:
jobs:
- start_pre-commit_tests:
@ -3098,3 +3328,9 @@ workflows:
requires:
- start_upgrade_tests
- build
- start_j8_dtests_offheap:
type: approval
- j8_dtests_offheap:
requires:
- start_j8_dtests_offheap
- build

242
.circleci/config.yml.MIDRES
View File

@ -495,6 +495,131 @@ jobs:
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_offheap_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: large
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 25
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Run repeated Python dtest
no_output_timeout: 15m
command: |
if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
echo "Running ${REPEATED_DTESTS} $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
java -version
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
echo "env: $(env)"
echo "** done env"
mkdir -p /tmp/results/dtests
tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
stop_on_failure_arg=""
if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
if true; then
vnodes_args="--use-vnodes --num-tokens=16"
fi
upgrade_arg=""
if false; then
upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir --use-off-heap-memtables --skip-resource-intensive-tests $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
utests_long_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -939,7 +1064,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1164,7 +1289,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -1647,6 +1772,105 @@ jobs:
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests_offheap:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: large
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 50
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
pip3 freeze
- run:
name: Determine Tests to Run (j8_dtests_offheap)
no_output_timeout: 5m
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_dtests_offheap)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --use-off-heap-memtables --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_dtests_offheap_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_dtests_offheap_raw /tmp/all_dtest_tests_j8_dtests_offheap\nelse\n grep -e '' /tmp/all_dtest_tests_j8_dtests_offheap_raw > /tmp/all_dtest_tests_j8_dtests_offheap || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_dtests_offheap > /tmp/split_dtest_tests_j8_dtests_offheap.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n"
- run:
name: Run dtests (j8_dtests_offheap)
no_output_timeout: 15m
command: "echo \"cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\"\ncat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_dtests_offheap_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_dtests_offheap.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
destination: dtest_j8_dtests_offheap
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_j8_dtests_offheap_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_TESTS_STOP_ON_FAILURE: false
- REPEATED_UTESTS: null
- REPEATED_UTESTS_COUNT: 500
- REPEATED_UTESTS_LONG: null
- REPEATED_UTESTS_LONG_COUNT: 100
- REPEATED_UTESTS_STRESS: null
- REPEATED_UTESTS_STRESS_COUNT: 500
- REPEATED_JVM_DTESTS: null
- REPEATED_JVM_DTESTS_COUNT: 500
- REPEATED_JVM_UPGRADE_DTESTS: null
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- REPEATED_DTESTS: null
- REPEATED_DTESTS_COUNT: 500
- REPEATED_UPGRADE_DTESTS: null
- REPEATED_UPGRADE_DTESTS_COUNT: 25
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_unit_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -2351,7 +2575,7 @@ jobs:
fi
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
@ -3040,6 +3264,12 @@ workflows:
requires:
- start_upgrade_dtests
- build
- start_j8_dtests_offheap:
type: approval
- j8_dtests_offheap:
requires:
- start_j8_dtests_offheap
- build
pre-commit_tests:
jobs:
- start_pre-commit_tests:
@ -3098,3 +3328,9 @@ workflows:
requires:
- start_upgrade_tests
- build
- start_j8_dtests_offheap:
type: approval
- j8_dtests_offheap:
requires:
- start_j8_dtests_offheap
- build

1
.circleci/generate.sh
View File

@ -275,6 +275,7 @@ delete_repeated_jobs()
if (! (echo "$env_vars" | grep -q "REPEATED_DTESTS=")); then
delete_job "$1" "j8_dtests_repeat"
delete_job "$1" "j8_dtests_vnode_repeat"
delete_job "$1" "j8_dtests_offheap_repeat"
delete_job "$1" "j11_dtests_repeat"
delete_job "$1" "j11_dtests_vnode_repeat"
fi