natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-4.0' into trunk

This commit is contained in:
David Capwell 2022-04-12 11:54:09 -07:00
parent 20175bf77e 1ebdc7e758
commit 6013f16de7
23 changed files with 1494 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.build/build-resolver.xml
View File

@ -56,6 +56,8 @@
<resolver:remoterepos id="all">
<remoterepo id="resolver-central" url="${artifact.remoteRepository.central}"/>
<remoterepo id="resolver-apache" url="${artifact.remoteRepository.apache}"/>
<!-- Snapshots are not allowed, but for feature branches they may be needed, so uncomment the below to allow snapshots to work -->
<!-- <remoterepo id="resolver-apache-snapshot" url="https://repository.apache.org/content/repositories/snapshots" releases="false" snapshots="true" updates="always" checksums="fail" /> -->
</resolver:remoterepos>
<macrodef name="resolve">

47
.circleci/config-2_1.yml
View File

@ -196,6 +196,10 @@ j8_with_dtests_jobs: &j8_with_dtests_jobs
requires:
- start_j8_jvm_dtests
- j8_build
- j8_jvm_dtests_vnode:
requires:
- start_j8_jvm_dtests
- j8_build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
@ -376,6 +380,9 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs
- j8_jvm_dtests:
requires:
- j8_build
- j8_jvm_dtests_vnode:
requires:
- j8_build
- j8_cqlshlib_tests:
requires:
- j8_build
@ -536,6 +543,10 @@ j11_with_dtests_jobs: &j11_with_dtests_jobs
requires:
- start_j11_jvm_dtests
- j11_build
- j11_jvm_dtests_vnode:
requires:
- start_j11_jvm_dtests
- j11_build
- start_j11_cqlshlib_tests:
type: approval
- j11_cqlshlib_tests:
@ -599,6 +610,9 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs
- j11_jvm_dtests:
requires:
- j11_build
- j11_jvm_dtests_vnode:
requires:
- j11_build
- j11_cqlshlib_tests:
requires:
- j11_build
@ -736,6 +750,20 @@ jobs:
classlistprefix: distributed
target: "testclasslist"
j8_jvm_dtests_vnode:
<<: *j8_small_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- create_junit_containers:
classlistprefix: distributed
extra_filters: "| grep -v upgrade"
- log_environment
- run_parallel_junit_tests:
classlistprefix: distributed
target: "testclasslist"
arguments: "-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
j11_jvm_dtests:
<<: *j11_small_par_executor
steps:
@ -749,6 +777,20 @@ jobs:
classlistprefix: distributed
target: "testclasslist"
j11_jvm_dtests_vnode:
<<: *j11_small_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- create_junit_containers:
classlistprefix: distributed
extra_filters: "| grep -v upgrade"
- log_environment
- run_parallel_junit_tests:
classlistprefix: distributed
target: "testclasslist"
arguments: "-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
j8_jvm_upgrade_dtests:
<<: *j8_medium_par_executor
steps:
@ -1322,6 +1364,9 @@ commands:
classlistprefix:
type: string
default: unit
arguments:
type: string
default: " "
steps:
- run:
name: Run Unit Tests (<<parameters.target>>)
@ -1337,7 +1382,7 @@ commands:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant <<parameters.target>> -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<<parameters.classlistprefix>>
ant <<parameters.target>> <<parameters.arguments>> -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<<parameters.classlistprefix>>
no_output_timeout: <<parameters.no_output_timeout>>
- store_test_results:
path: /tmp/cassandra/build/test/output/

253
.circleci/config.yml
View File

@ -39,7 +39,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -83,7 +83,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -302,6 +302,115 @@ jobs:
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210929
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (testclasslist)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j11_unit_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:20210304
@ -367,7 +476,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -917,6 +1026,116 @@ jobs:
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j11_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:20210304
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (testclasslist)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j8_cqlsh-dtests-py3-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210929
@ -1658,7 +1877,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -1723,7 +1942,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -1767,7 +1986,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2335,7 +2554,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -2379,7 +2598,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2839,7 +3058,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -3164,7 +3383,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -3499,6 +3718,10 @@ workflows:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_jvm_dtests_vnode:
requires:
- start_j8_jvm_dtests
- j8_build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
@ -3664,6 +3887,9 @@ workflows:
- j8_jvm_dtests:
requires:
- j8_build
- j8_jvm_dtests_vnode:
requires:
- j8_build
- j8_cqlshlib_tests:
requires:
- j8_build
@ -3809,6 +4035,10 @@ workflows:
requires:
- start_j11_jvm_dtests
- j11_build
- j11_jvm_dtests_vnode:
requires:
- start_j11_jvm_dtests
- j11_build
- start_j11_cqlshlib_tests:
type: approval
- j11_cqlshlib_tests:
@ -3868,6 +4098,9 @@ workflows:
- j11_jvm_dtests:
requires:
- j11_build
- j11_jvm_dtests_vnode:
requires:
- j11_build
- j11_cqlshlib_tests:
requires:
- j11_build

253
.circleci/config.yml.HIGHRES
View File

@ -39,7 +39,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -83,7 +83,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -302,6 +302,115 @@ jobs:
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210929
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 5
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (testclasslist)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j11_unit_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:20210304
@ -367,7 +476,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -917,6 +1026,116 @@ jobs:
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j11_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:20210304
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 5
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (testclasslist)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j8_cqlsh-dtests-py3-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210929
@ -1658,7 +1877,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -1723,7 +1942,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -1767,7 +1986,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2335,7 +2554,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -2379,7 +2598,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2839,7 +3058,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -3164,7 +3383,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -3499,6 +3718,10 @@ workflows:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_jvm_dtests_vnode:
requires:
- start_j8_jvm_dtests
- j8_build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
@ -3664,6 +3887,9 @@ workflows:
- j8_jvm_dtests:
requires:
- j8_build
- j8_jvm_dtests_vnode:
requires:
- j8_build
- j8_cqlshlib_tests:
requires:
- j8_build
@ -3809,6 +4035,10 @@ workflows:
requires:
- start_j11_jvm_dtests
- j11_build
- j11_jvm_dtests_vnode:
requires:
- start_j11_jvm_dtests
- j11_build
- start_j11_cqlshlib_tests:
type: approval
- j11_cqlshlib_tests:
@ -3868,6 +4098,9 @@ workflows:
- j11_jvm_dtests:
requires:
- j11_build
- j11_jvm_dtests_vnode:
requires:
- j11_build
- j11_cqlshlib_tests:
requires:
- j11_build

253
.circleci/config.yml.LOWRES
View File

@ -39,7 +39,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -83,7 +83,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -302,6 +302,115 @@ jobs:
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210929
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (testclasslist)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j11_unit_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:20210304
@ -367,7 +476,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -917,6 +1026,116 @@ jobs:
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j11_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:20210304
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (testclasslist)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j8_cqlsh-dtests-py3-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210929
@ -1658,7 +1877,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -1723,7 +1942,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -1767,7 +1986,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2335,7 +2554,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -2379,7 +2598,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2839,7 +3058,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -3164,7 +3383,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -3499,6 +3718,10 @@ workflows:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_jvm_dtests_vnode:
requires:
- start_j8_jvm_dtests
- j8_build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
@ -3664,6 +3887,9 @@ workflows:
- j8_jvm_dtests:
requires:
- j8_build
- j8_jvm_dtests_vnode:
requires:
- j8_build
- j8_cqlshlib_tests:
requires:
- j8_build
@ -3809,6 +4035,10 @@ workflows:
requires:
- start_j11_jvm_dtests
- j11_build
- j11_jvm_dtests_vnode:
requires:
- start_j11_jvm_dtests
- j11_build
- start_j11_cqlshlib_tests:
type: approval
- j11_cqlshlib_tests:
@ -3868,6 +4098,9 @@ workflows:
- j11_jvm_dtests:
requires:
- j11_build
- j11_jvm_dtests_vnode:
requires:
- j11_build
- j11_cqlshlib_tests:
requires:
- j11_build

261
.circleci/config.yml.MIDRES
View File

@ -39,7 +39,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -83,7 +83,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -302,6 +302,115 @@ jobs:
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210929
resource_class: large
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 10
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (testclasslist)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j11_unit_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:20210304
@ -367,7 +476,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -917,6 +1026,116 @@ jobs:
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j11_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:20210304
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 10
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Unit Tests (testclasslist)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j8_cqlsh-dtests-py3-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210929
@ -1658,7 +1877,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -1723,7 +1942,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -1767,7 +1986,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2145,10 +2364,10 @@ jobs:
j8_dtests-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210929
resource_class: large
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 50
parallelism: 25
steps:
- attach_workspace:
at: /home/cassandra
@ -2335,7 +2554,7 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" "$HOME/cassandra/test/simulator/test/**/*.java" > /tmp/all_java_unit_tests.txt
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -2379,7 +2598,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -2839,7 +3058,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -3102,10 +3321,10 @@ jobs:
utests_compression:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210929
resource_class: medium
resource_class: large
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 25
parallelism: 50
steps:
- attach_workspace:
at: /home/cassandra
@ -3164,7 +3383,7 @@ jobs:
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@ -3499,6 +3718,10 @@ workflows:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_jvm_dtests_vnode:
requires:
- start_j8_jvm_dtests
- j8_build
- start_j8_cqlshlib_tests:
type: approval
- j8_cqlshlib_tests:
@ -3664,6 +3887,9 @@ workflows:
- j8_jvm_dtests:
requires:
- j8_build
- j8_jvm_dtests_vnode:
requires:
- j8_build
- j8_cqlshlib_tests:
requires:
- j8_build
@ -3809,6 +4035,10 @@ workflows:
requires:
- start_j11_jvm_dtests
- j11_build
- j11_jvm_dtests_vnode:
requires:
- start_j11_jvm_dtests
- j11_build
- start_j11_cqlshlib_tests:
type: approval
- j11_cqlshlib_tests:
@ -3868,6 +4098,9 @@ workflows:
- j11_jvm_dtests:
requires:
- j11_build
- j11_jvm_dtests_vnode:
requires:
- j11_build
- j11_cqlshlib_tests:
requires:
- j11_build

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.1
* Add support for vnodes in jvm-dtest (CASSANDRA-17332)
* Remove guardrails global enable flag (CASSANDRA-17499)
* Clients using JMX are unable to handle non-standard java types but we leak this into our interfaces (CASSANDRA-17527)
* Remove stress server functionality (CASSANDRA-17535)

10
build.xml
View File

@ -78,6 +78,7 @@
<property name="test.simulator-test.src" value="${test.dir}/simulator/test"/>
<property name="test.driver.connection_timeout_ms" value="5000"/>
<property name="test.driver.read_timeout_ms" value="12000"/>
<property name="test.jvm.args" value="" />
<property name="dist.dir" value="${build.dir}/dist"/>
<property name="tmp.dir" value="${java.io.tmpdir}"/>
@ -144,6 +145,8 @@
<property name="chronicle-wire.version" value="2.20.117" />
<property name="chronicle-threads.version" value="2.20.111" />
<property name="dtest-api.version" value="0.0.13" />
<condition property="maven-ant-tasks.jar.exists">
<available file="${build.dir}/maven-ant-tasks-${maven-ant-tasks.version}.jar" />
</condition>
@ -242,7 +245,7 @@
<string>-Dio.netty.tryReflectionSetAccessible=true</string>
</resources>
<pathconvert property="_jvm11_test_arg_items_concat" refid="_jvm11_test_arg_items" pathsep=" "/>
<condition property="test-jvmargs" value="${_jvm11_test_arg_items_concat}" else="${_jvm8_test_arg_items_concat}">
<condition property="_std-test-jvmargs" value="${_jvm11_test_arg_items_concat}" else="${_jvm8_test_arg_items_concat}">
<not>
<equals arg1="${ant.java.version}" arg2="1.8"/>
</not>
@ -548,9 +551,9 @@
<dependency groupId="com.google.code.java-allocation-instrumenter" artifactId="java-allocation-instrumenter" version="${allocation-instrumenter.version}" scope="test">
<exclusion groupId="com.google.guava" artifactId="guava"/>
</dependency>
<dependency groupId="org.apache.cassandra" artifactId="dtest-api" version="0.0.12" scope="test"/>
<dependency groupId="org.apache.cassandra" artifactId="harry-core" version="0.0.1" scope="test"/>
<dependency groupId="org.reflections" artifactId="reflections" version="0.10.2" scope="test"/>
<dependency groupId="org.apache.cassandra" artifactId="dtest-api" version="${dtest-api.version}" scope="test"/>
<dependency groupId="com.puppycrawl.tools" artifactId="checkstyle" version="8.40" scope="test"/>
<dependency groupId="org.apache.hadoop" artifactId="hadoop-core" version="1.0.3" scope="provided">
<exclusion groupId="org.mortbay.jetty" artifactId="servlet-api"/>
@ -1501,7 +1504,8 @@
<jvmarg line="${java11-jvmargs}"/>
<!-- disable shrinks in quicktheories CASSANDRA-15554 -->
<jvmarg value="-DQT_SHRINKS=0"/>
<jvmarg line="${test-jvmargs}" />
<jvmarg line="${_std-test-jvmargs}" />
<jvmarg line="${test.jvm.args}" />
<optjvmargs/>
<!-- Uncomment to debug unittest, attach debugger to port 1416 -->
<!--

8
test/distributed/org/apache/cassandra/distributed/action/GossipHelper.java
View File

@ -30,6 +30,8 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
@ -370,9 +372,11 @@ public class GossipHelper
{
return instance.appliesOnInstance((String partitionerString, String tokenString) -> {
IPartitioner partitioner = FBUtilities.newPartitioner(partitionerString);
Token token = partitioner.getTokenFactory().fromString(tokenString);
Collection<Token> tokens = tokenString.contains(",")
? Stream.of(tokenString.split(",")).map(partitioner.getTokenFactory()::fromString).collect(Collectors.toList())
: Collections.singleton(partitioner.getTokenFactory().fromString(tokenString));
VersionedValue versionedValue = supplier.apply(partitioner, Collections.singleton(token));
VersionedValue versionedValue = supplier.apply(partitioner, tokens);
return new VersionedApplicationState(applicationState.ordinal(), versionedValue.value, versionedValue.version);
}).apply(partitionerStr, initialTokenStr);
}

77
test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
View File

@ -18,6 +18,7 @@
package org.apache.cassandra.distributed.impl;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
@ -38,13 +39,11 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
import java.util.function.BiPredicate;
import java.util.function.Consumer;
import java.util.function.Function;
@ -54,8 +53,10 @@ import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.annotation.concurrent.GuardedBy;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.junit.Assume;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -201,6 +202,37 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
this.shutdownExecutor = shutdownExecutor;
return (B) this;
}
@Override
public C createWithoutStarting() throws IOException
{
// if running as vnode but test sets withoutVNodes(), then skip the test
// AbstractCluster.createInstanceConfig has similar logic, but handles the cases where the test
// attempts to control tokens via config
// when token supplier is defined, use getTokenCount() to see if vnodes is supported or not
if (isVnode())
{
Assume.assumeTrue("vnode is not supported", isVNodeAllowed());
// if token count > 1 and isVnode, then good
Assume.assumeTrue("no-vnode is requested but not supported", getTokenCount() > 1);
}
else
{
Assume.assumeTrue("single-token is not supported", isSingleTokenAllowed());
// if token count == 1 and isVnode == false, then goodAbstractClusterTest
Assume.assumeTrue("vnode is requested but not supported", getTokenCount() == 1);
}
return super.createWithoutStarting();
}
private boolean isVnode()
{
TokenSupplier ts = getTokenSupplier();
return ts == null
? getTokenCount() > 1 // token supplier wasn't defined yet, so rely on getTokenCount()
: ts.tokens(1).size() > 1; // token supplier is defined... check the first instance to see what tokens are used
}
}
protected class Wrapper extends DelegatingInvokableInstance implements IUpgradeableInstance
@ -516,15 +548,46 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
return createInstanceConfig(size() + 1);
}
private InstanceConfig createInstanceConfig(int nodeNum)
@VisibleForTesting
InstanceConfig createInstanceConfig(int nodeNum)
{
INodeProvisionStrategy provisionStrategy = nodeProvisionStrategy.create(subnet);
long token = tokenSupplier.token(nodeNum);
Collection<String> tokens = tokenSupplier.tokens(nodeNum);
NetworkTopology topology = buildNetworkTopology(provisionStrategy, nodeIdTopology);
InstanceConfig config = InstanceConfig.generate(nodeNum, provisionStrategy, topology, root, Long.toString(token), datadirCount);
InstanceConfig config = InstanceConfig.generate(nodeNum, provisionStrategy, topology, root, tokens, datadirCount);
config.set(Constants.KEY_DTEST_API_CLUSTER_ID, clusterId.toString());
// if a test sets num_tokens directly, then respect it and only run if vnode or no-vnode is defined
int defaultTokenCount = config.getInt("num_tokens");
assert tokens.size() == defaultTokenCount : String.format("num_tokens=%d but tokens are %s; size does not match", defaultTokenCount, tokens);
String defaultTokens = config.getString("initial_token");
if (configUpdater != null)
{
configUpdater.accept(config);
int testTokenCount = config.getInt("num_tokens");
if (defaultTokenCount != testTokenCount)
{
if (testTokenCount == 1)
{
// test is no-vnode, but running with vnode, so skip
Assume.assumeTrue("vnode is not supported", false);
}
else
{
Assume.assumeTrue("no-vnode is requested but not supported", defaultTokenCount > 1);
// if the test controls initial_token or GOSSIP is enabled, then the test is safe to run
if (defaultTokens.equals(config.getString("initial_token")))
{
// test didn't define initial_token
Assume.assumeTrue("vnode is enabled and num_tokens is defined in test without GOSSIP or setting initial_token", config.has(Feature.GOSSIP));
config.remove("initial_token");
}
else
{
// test defined initial_token; trust it
}
}
}
}
return config;
}
@ -1025,12 +1088,12 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
public List<Token> tokens()
{
return stream()
.map(i ->
.flatMap(i ->
{
try
{
IPartitioner partitioner = ((IPartitioner)Class.forName(i.config().getString("partitioner")).newInstance());
return partitioner.getTokenFactory().fromString(i.config().getString("initial_token"));
return Stream.of(i.config().getString("initial_token").split(",")).map(partitioner.getTokenFactory()::fromString);
}
catch (Throwable t)
{

251
test/distributed/org/apache/cassandra/distributed/impl/AbstractClusterTest.java Normal file
View File

@ -0,0 +1,251 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.impl;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Consumer;
import org.junit.AssumptionViolatedException;
import org.junit.Test;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.impl.AbstractCluster.AbstractBuilder;
import org.apache.cassandra.distributed.shared.Versions;
import org.apache.cassandra.utils.FailingRunnable;
import org.assertj.core.api.Assertions;
import org.mockito.Mockito;
public class AbstractClusterTest
{
@Test
public void allowVnodeWithMultipleTokens()
{
AbstractBuilder builder = builder();
builder.withTokenCount(42);
unroll(() -> builder.createWithoutStarting());
}
@Test
public void allowVnodeWithSingleToken()
{
AbstractBuilder builder = builder();
builder.withTokenCount(1);
unroll(() -> builder.createWithoutStarting());
}
@Test
public void disallowVnodeWithMultipleTokens()
{
AbstractBuilder builder = builder();
builder.withoutVNodes();
builder.withTokenCount(42);
Assertions.assertThatThrownBy(() -> builder.createWithoutStarting())
.isInstanceOf(AssumptionViolatedException.class)
.hasMessage("vnode is not supported");
}
@Test
public void disallowVnodeWithSingleToken()
{
AbstractBuilder builder = builder();
builder.withoutVNodes();
builder.withTokenCount(1);
unroll(() -> builder.createWithoutStarting());
}
@Test
public void withoutVNodes()
{
AbstractBuilder builder = builder();
builder.withoutVNodes();
//TODO casting is annoying... what can be done to be smarter?
builder.withTokenSupplier((TokenSupplier) i -> Arrays.asList("a", "b", "c"));
Assertions.assertThatThrownBy(() -> builder.createWithoutStarting())
.isInstanceOf(AssumptionViolatedException.class)
.hasMessage("vnode is not supported");
}
@Test
public void vnodeButTokensDoNotMatch()
{
AbstractBuilder builder = builder();
builder.withTokenCount(1);
//TODO casting is annoying... what can be done to be smarter?
builder.withTokenSupplier((TokenSupplier) i -> Arrays.asList("a", "b", "c"));
Assertions.assertThatThrownBy(() -> builder.createWithoutStarting())
.isInstanceOf(AssumptionViolatedException.class)
.hasMessage("no-vnode is requested but not supported");
}
@Test
public void noVnodeButTokensDoNotMatch()
{
AbstractBuilder builder = builder();
builder.withTokenCount(42);
//TODO casting is annoying... what can be done to be smarter?
builder.withTokenSupplier((TokenSupplier) i -> Arrays.asList("a"));
Assertions.assertThatThrownBy(() -> builder.createWithoutStarting())
.isInstanceOf(AssumptionViolatedException.class)
.hasMessage("vnode is requested but not supported");
}
@Test
public void vnodeNotSupported()
{
ConfigUpdate config = ConfigUpdate.of("num_tokens", 1);
AbstractCluster<?> cluster = cluster(4, config);
config.check = true;
Assertions.assertThatThrownBy(() -> cluster.createInstanceConfig(1))
.isInstanceOf(AssumptionViolatedException.class)
.hasMessage("vnode is not supported");
}
@Test
public void noVnodeNotSupported()
{
ConfigUpdate config = ConfigUpdate.of("num_tokens", 4);
AbstractCluster<?> cluster = cluster(1, config);
config.check = true;
Assertions.assertThatThrownBy(() -> cluster.createInstanceConfig(1))
.isInstanceOf(AssumptionViolatedException.class)
.hasMessage("no-vnode is requested but not supported");
}
@Test
public void vnodeMismatch()
{
ConfigUpdate config = ConfigUpdate.of("num_tokens", 4);
AbstractCluster<?> cluster = cluster(2, config);
config.check = true;
Assertions.assertThatThrownBy(() -> cluster.createInstanceConfig(1))
.isInstanceOf(AssumptionViolatedException.class)
.hasMessage("vnode is enabled and num_tokens is defined in test without GOSSIP or setting initial_token");
}
@Test
public void vnodeMismatchDefinesTokens()
{
ConfigUpdate config = ConfigUpdate.of("num_tokens", 4, "initial_token", "some values");
AbstractCluster<?> cluster = cluster(2, config);
config.check = true;
unroll(() -> cluster.createInstanceConfig(1));
}
private static void unroll(FailingRunnable r)
{
try
{
r.run();
}
catch (AssumptionViolatedException e)
{
AssertionError e2 = new AssertionError(e.getMessage());
e2.setStackTrace(e.getStackTrace());
throw e2;
}
}
private static AbstractCluster<?> cluster(int tokenCount, Consumer<IInstanceConfig> fn)
{
try
{
return (AbstractCluster<?>) builder()
.withTokenCount(tokenCount)
.withConfig(fn)
.createWithoutStarting();
}
catch (IOException e)
{
throw new AssertionError(e);
}
}
private static AbstractBuilder builder()
{
return new Builder().withNodes(1);
}
private static class Builder extends AbstractBuilder<IInvokableInstance, Cluster, Builder>
{
public Builder()
{
super(Cluster::new);
}
}
private static class Cluster extends AbstractCluster<IInvokableInstance>
{
protected Cluster(AbstractBuilder builder)
{
super(builder);
}
@Override
protected IInvokableInstance newInstanceWrapper(Versions.Version version, IInstanceConfig config)
{
IInvokableInstance inst = Mockito.mock(IInvokableInstance.class);
Mockito.when(inst.config()).thenReturn(config);
return inst;
}
}
private static class ConfigUpdate implements Consumer<IInstanceConfig>
{
public boolean check = false;
private final Map<String, Object> override;
private ConfigUpdate(Map<String, Object> override)
{
this.override = override;
}
public static ConfigUpdate of(Object... args)
{
Map<String, Object> override = new HashMap<>();
for (int i = 0; i < args.length; i = i + 2)
override.put((String) args[i], args[i + 1]);
return new ConfigUpdate(override);
}
@Override
public void accept(IInstanceConfig config)
{
if (!check)
return;
for (Map.Entry<String, Object> e : override.entrySet())
config.set(e.getKey(), e.getValue());
}
}
}

25
test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java
View File

@ -21,12 +21,14 @@ package org.apache.cassandra.distributed.impl;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Map;
import java.util.TreeMap;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.vdurmont.semver4j.Semver;
@ -67,14 +69,16 @@ public class InstanceConfig implements IInstanceConfig
String commitlog_directory,
String hints_directory,
String cdc_raw_directory,
String initial_token,
Collection<String> initial_token,
int storage_port,
int native_transport_port)
{
this.num = num;
this.networkTopology = networkTopology;
this.hostId = new UUID(0x4000L, (1L << 63) | num); // deterministic hostId for simulator
this .set("num_tokens", 1)
//TODO move away from magic strings in favor of constants
this .set("num_tokens", initial_token.size())
.set("initial_token", initial_token.stream().collect(Collectors.joining(",")))
.set("broadcast_address", broadcast_address)
.set("listen_address", listen_address)
.set("broadcast_rpc_address", broadcast_rpc_address)
@ -84,7 +88,6 @@ public class InstanceConfig implements IInstanceConfig
.set("commitlog_directory", commitlog_directory)
.set("hints_directory", hints_directory)
.set("cdc_raw_directory", cdc_raw_directory)
.set("initial_token", initial_token)
.set("partitioner", "org.apache.cassandra.dht.Murmur3Partitioner")
.set("start_native_transport", true)
.set("concurrent_writes", 2)
@ -189,6 +192,12 @@ public class InstanceConfig implements IInstanceConfig
return this;
}
public InstanceConfig remove(String fieldName)
{
getParams(fieldName).remove(fieldName);
return this;
}
public InstanceConfig forceSet(String fieldName, Object value)
{
getParams(fieldName).put(fieldName, value);
@ -208,10 +217,12 @@ public class InstanceConfig implements IInstanceConfig
throw new IllegalStateException("In-JVM dtests no longer support propagate");
}
@Override
public void validate()
{
if (((int) get("num_tokens")) > 1)
throw new IllegalArgumentException("In-JVM dtests do not support vnodes as of now.");
// Previous logic would validate vnode was not used, but with vnode support added that validation isn't needed.
// Rather than attempting validating the configs here, its best to leave that to the instance; this method
// is no longer really needed, but can not be removed due to backwards compatability.
}
public Object get(String name)
@ -238,7 +249,7 @@ public class InstanceConfig implements IInstanceConfig
INodeProvisionStrategy provisionStrategy,
NetworkTopology networkTopology,
Path root,
String token,
Collection<String> tokens,
int datadirCount)
{
return new InstanceConfig(nodeNum,
@ -254,7 +265,7 @@ public class InstanceConfig implements IInstanceConfig
String.format("%s/node%d/commitlog", root, nodeNum),
String.format("%s/node%d/hints", root, nodeNum),
String.format("%s/node%d/cdc", root, nodeNum),
token,
tokens,
provisionStrategy.storagePort(nodeNum),
provisionStrategy.nativeTransportPort(nodeNum));
}

17
test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
View File

@ -275,15 +275,13 @@ public class ClusterUtils
.collect(Collectors.toList()));
}
public static String getLocalToken(IInvokableInstance inst)
public static Collection<String> getLocalTokens(IInvokableInstance inst)
{
return inst.callOnInstance(() -> {
List<String> tokens = new ArrayList<>();
for (Token t : StorageService.instance.getTokenMetadata().getTokens(FBUtilities.getBroadcastAddressAndPort()))
tokens.add(t.getTokenValue().toString());
assert tokens.size() == 1 : "getLocalToken assumes a single token, but multiple tokens found";
return tokens.get(0);
return tokens;
});
}
@ -629,6 +627,17 @@ public class ClusterUtils
return Arrays.asList(token);
}
/**
* Get the number of tokens for the instance via config.
*
* @param instance to get token count from
* @return number of tokens
*/
public static int getTokenCount(IInvokableInstance instance)
{
return instance.config().getInt("num_tokens");
}
/**
* Get all data directories for the given instance.
*

5
test/distributed/org/apache/cassandra/distributed/test/CASTest.java
View File

@ -79,8 +79,9 @@ public class CASTest extends CASCommonTestCases
.set("write_request_timeout", REQUEST_TIMEOUT)
.set("cas_contention_timeout", CONTENTION_TIMEOUT)
.set("request_timeout", REQUEST_TIMEOUT);
THREE_NODES = init(Cluster.create(3, conf));
FOUR_NODES = init(Cluster.create(4, conf), 3);
// TODO: fails with vnode enabled
THREE_NODES = init(Cluster.build(3).withConfig(conf).withoutVNodes().start());
FOUR_NODES = init(Cluster.build(4).withConfig(conf).withoutVNodes().start(), 3);
}
@AfterClass

12
test/distributed/org/apache/cassandra/distributed/test/GossipTest.java
View File

@ -37,6 +37,7 @@ import net.bytebuddy.implementation.MethodDelegation;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.*;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.EndpointState;
import org.apache.cassandra.gms.Gossiper;
@ -46,13 +47,13 @@ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.utils.FBUtilities;
import org.assertj.core.api.Assertions;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.apache.cassandra.distributed.impl.DistributedTestSnitch.toCassandraInetAddressAndPort;
import static org.apache.cassandra.distributed.shared.ClusterUtils.getLocalToken;
import static org.apache.cassandra.distributed.shared.ClusterUtils.runAndWaitForLogs;
import static org.junit.Assert.assertEquals;
@ -239,9 +240,11 @@ public class GossipTest extends TestBaseImpl
@Test
public void gossipShutdownUpdatesTokenMetadata() throws Exception
{
// TODO: fails with vnode enabled
try (Cluster cluster = Cluster.build(3)
.withConfig(c -> c.with(Feature.GOSSIP, Feature.NETWORK))
.withInstanceInitializer(FailureHelper::installMoveFailure)
.withoutVNodes()
.start())
{
init(cluster, 2);
@ -277,6 +280,13 @@ public class GossipTest extends TestBaseImpl
}
}
private static String getLocalToken(IInvokableInstance node)
{
Collection<String> tokens = ClusterUtils.getLocalTokens(node);
Assertions.assertThat(tokens).hasSize(1);
return tokens.stream().findFirst().get();
}
void assertPendingRangesForPeer(final boolean expectPending, final InetSocketAddress movingAddress, final Cluster cluster)
{
for (IInvokableInstance inst : new IInvokableInstance[]{ cluster.get(1), cluster.get(3)})

2
test/distributed/org/apache/cassandra/distributed/test/MoveTest.java
View File

@ -47,8 +47,10 @@ public class MoveTest extends TestBaseImpl
private void move(boolean forwards) throws Throwable
{
// TODO: fails with vnode enabled
try (Cluster cluster = Cluster.build(4)
.withConfig(config -> config.set("paxos_variant", "v2_without_linearizable_reads").with(NETWORK).with(GOSSIP))
.withoutVNodes()
.start())
{
cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};");

51
test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java
View File

@ -215,7 +215,8 @@ public class PaxosRepairTest extends TestBaseImpl
@Test
public void paxosRepairTest() throws Throwable
{
try (Cluster cluster = init(Cluster.create(3, CONFIG_CONSUMER)))
// TODO: fails with vnode enabled
try (Cluster cluster = init(Cluster.build(3).withConfig(CONFIG_CONSUMER).withoutVNodes().start()))
{
cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM);
@ -264,7 +265,8 @@ public class PaxosRepairTest extends TestBaseImpl
@Test
public void topologyChangePaxosTest() throws Throwable
{
try (Cluster cluster = Cluster.build(4).withConfig(CONFIG_CONSUMER).createWithoutStarting())
// TODO: fails with vnode enabled
try (Cluster cluster = Cluster.build(4).withConfig(CONFIG_CONSUMER).withoutVNodes().createWithoutStarting())
{
for (int i=1; i<=3; i++)
cluster.get(i).startup();
@ -296,11 +298,15 @@ public class PaxosRepairTest extends TestBaseImpl
@Test
public void paxosCleanupWithReproposal() throws Throwable
{
try (Cluster cluster = init(Cluster.create(3, cfg -> cfg
.set("paxos_variant", "v2")
.set("paxos_purge_grace_period", 0)
.set("paxos_state_purging", Config.PaxosStatePurging.repaired.toString())
.set("truncate_request_timeout_in_ms", 1000L))))
// TODO: fails with vnode enabled
try (Cluster cluster = init(Cluster.build(3)
.withConfig(cfg -> cfg
.set("paxos_variant", "v2")
.set("paxos_purge_grace_period", 0)
.set("paxos_state_purging", Config.PaxosStatePurging.repaired.toString())
.set("truncate_request_timeout_in_ms", 1000L))
.withoutVNodes()
.start()))
{
cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
@ -363,11 +369,15 @@ public class PaxosRepairTest extends TestBaseImpl
@Test
public void paxosCleanupWithReproposalClashingTimestamp() throws Throwable
{
try (Cluster cluster = init(Cluster.create(5, cfg -> cfg
.set("paxos_variant", "v2")
.set("paxos_purge_grace_period", 0)
.set("paxos_cache_size", "0")
.set("truncate_request_timeout_in_ms", 1000L))))
// TODO: fails with vnode enabled
try (Cluster cluster = init(Cluster.build(5)
.withConfig(cfg -> cfg
.set("paxos_variant", "v2")
.set("paxos_purge_grace_period", 0)
.set("paxos_cache_size", "0")
.set("truncate_request_timeout_in_ms", 1000L))
.withoutVNodes()
.start()))
{
cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
@ -431,11 +441,15 @@ public class PaxosRepairTest extends TestBaseImpl
@Test
public void paxosCleanupWithDelayedProposal() throws Throwable
{
try (Cluster cluster = init(Cluster.create(3, cfg -> cfg
.set("paxos_variant", "v2")
.set("paxos_purge_grace_period", 0)
.set("paxos_state_purging", Config.PaxosStatePurging.repaired.toString())
.set("truncate_request_timeout_in_ms", 1000L)))
// TODO: fails with vnode enabled
try (Cluster cluster = init(Cluster.build(3)
.withConfig(cfg -> cfg
.set("paxos_variant", "v2")
.set("paxos_purge_grace_period", 0)
.set("paxos_state_purging", Config.PaxosStatePurging.repaired.toString())
.set("truncate_request_timeout_in_ms", 1000L))
.withoutVNodes()
.start())
)
{
cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
@ -516,7 +530,8 @@ public class PaxosRepairTest extends TestBaseImpl
@Test
public void paxosRepairVersionGate() throws Throwable
{
try (Cluster cluster = init(Cluster.create(3, CONFIG_CONSUMER)))
// TODO: fails with vnode enabled
try (Cluster cluster = init(Cluster.build(3).withConfig(CONFIG_CONSUMER).withoutVNodes().start()))
{
cluster.schemaChange("CREATE TABLE " + KEYSPACE + '.' + TABLE + " (pk int, ck int, v int, PRIMARY KEY (pk, ck))");
cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + '.' + TABLE + " (pk, ck, v) VALUES (1, 1, 1) IF NOT EXISTS", ConsistencyLevel.QUORUM);

7
test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
View File

@ -53,6 +53,7 @@ import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.distributed.api.IMessage;
import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.distributed.shared.RepairResult;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.Verb;
@ -266,6 +267,7 @@ public class PreviewRepairTest extends TestBaseImpl
ExecutorService es = Executors.newSingleThreadExecutor();
try(Cluster cluster = init(Cluster.build(2).withConfig(config -> config.with(GOSSIP).with(NETWORK)).start()))
{
int tokenCount = ClusterUtils.getTokenCount(cluster.get(1));
cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int)");
insert(cluster.coordinator(1), 0, 100);
@ -289,7 +291,7 @@ public class PreviewRepairTest extends TestBaseImpl
return res;
});
assertEquals(2, localRanges.size());
assertEquals(2 * tokenCount, localRanges.size());
Future<RepairResult> repairStatusFuture = es.submit(() -> cluster.get(1).callOnInstance(repair(options(true, false, localRanges.get(0)))));
previewRepairStarted.await(); // wait for node1 to start validation compaction
// this needs to finish before the preview repair is unpaused on node2
@ -318,6 +320,7 @@ public class PreviewRepairTest extends TestBaseImpl
.with(NETWORK))
.start()))
{
int tokenCount = ClusterUtils.getTokenCount(cluster.get(1));
cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int)");
insert(cluster.coordinator(1), 0, 100);
cluster.forEach((node) -> node.flush(KEYSPACE));
@ -341,7 +344,7 @@ public class PreviewRepairTest extends TestBaseImpl
return res;
});
assertEquals(2, localRanges.size());
assertEquals(2 * tokenCount, localRanges.size());
String [] previewedRange = localRanges.get(0).split(":");
String [] repairedRange = localRanges.get(1).split(":");
Future<NodeToolResult> repairStatusFuture = es.submit(() -> cluster.get(1).nodetoolResult("repair", "-st", repairedRange[0], "-et", repairedRange[1], KEYSPACE, "tbl"));

3
test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
View File

@ -180,7 +180,8 @@ public class ReadRepairTest extends TestBaseImpl
@Test
public void movingTokenReadRepairTest() throws Throwable
{
try (Cluster cluster = init(Cluster.create(4), 3))
// TODO: fails with vnode enabled
try (Cluster cluster = init(Cluster.build(4).withoutVNodes().start(), 3))
{
List<Token> tokens = cluster.tokens();

7
test/distributed/org/apache/cassandra/distributed/test/RepairTest.java
View File

@ -202,6 +202,7 @@ public class RepairTest extends TestBaseImpl
// The test uses its own keyspace with rf == 2
String forceRepairKeyspace = "test_force_repair_keyspace";
int rf = 2;
int tokenCount = ClusterUtils.getTokenCount(cluster.get(1));
cluster.schemaChange("CREATE KEYSPACE " + forceRepairKeyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': " + rf + "};");
try
@ -212,9 +213,9 @@ public class RepairTest extends TestBaseImpl
Set<String> requestedRanges = row.getSet("requested_ranges");
Assert.assertNotNull("Found no successful ranges", successfulRanges);
Assert.assertNotNull("Found no requested ranges", requestedRanges);
Assert.assertEquals("Requested ranges count should equals to replication factor", rf, requestedRanges.size());
Assert.assertTrue("Given clusterSize = 3, RF = 2 and 1 node down in the replica set, it should yield only 1 successful repaired range.",
successfulRanges.size() == 1 && !successfulRanges.contains("")); // the successful ranges set should not only contain empty string
Assert.assertEquals("Requested ranges count should equals to replication factor", rf * tokenCount, requestedRanges.size());
Assert.assertTrue("Given clusterSize = 3, RF = 2 and 1 node down in the replica set, it should yield only " + tokenCount + " successful repaired range.",
successfulRanges.size() == tokenCount && !successfulRanges.contains("")); // the successful ranges set should not only contain empty string
});
}
finally

8
test/distributed/org/apache/cassandra/distributed/test/ReplicaFilteringProtectionTest.java
View File

@ -22,6 +22,8 @@ import java.io.IOException;
import java.util.List;
import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.AssumptionViolatedException;
import org.junit.BeforeClass;
import org.junit.Test;
@ -57,8 +59,7 @@ public class ReplicaFilteringProtectionTest extends TestBaseImpl
cluster = init(Cluster.build()
.withNodes(REPLICAS)
.withConfig(config -> config.set("hinted_handoff_enabled", false)
.set("commitlog_sync", "batch")
.set("num_tokens", 1)).start());
.set("commitlog_sync", "batch")).start());
// Make sure we start w/ the correct defaults:
cluster.get(1).runOnInstance(() -> assertEquals(DEFAULT_WARN_THRESHOLD, StorageService.instance.getCachedReplicaRowsWarnThreshold()));
@ -68,7 +69,8 @@ public class ReplicaFilteringProtectionTest extends TestBaseImpl
@AfterClass
public static void teardown()
{
cluster.close();
if (cluster != null)
cluster.close();
}
@Test

24
test/unit/org/apache/cassandra/concurrent/DebuggableThreadPoolExecutorTest.java
View File

@ -22,12 +22,10 @@ package org.apache.cassandra.concurrent;
import java.lang.Thread.UncaughtExceptionHandler;
import java.util.UUID;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import com.google.common.base.Throwables;
import com.google.common.net.InetAddresses;
import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Assert;
@ -41,9 +39,8 @@ import org.apache.cassandra.tracing.TraceState;
import org.apache.cassandra.tracing.TraceStateImpl;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.FailingRunnable;
import org.apache.cassandra.utils.WrappedRunnable;
import org.assertj.core.api.Assertions;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
@ -321,23 +318,4 @@ public class DebuggableThreadPoolExecutorTest
private static final class DebuggingThrowsException extends RuntimeException {
}
// REVIEWER : I know this is the same as WrappedRunnable, but that doesn't support lambda...
private interface FailingRunnable extends Runnable
{
void doRun() throws Throwable;
default void run()
{
try
{
doRun();
}
catch (Throwable t)
{
Throwables.throwIfUnchecked(t);
throw new RuntimeException(t);
}
}
}
}

38
test/unit/org/apache/cassandra/utils/FailingRunnable.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils;
import com.google.common.base.Throwables;
public interface FailingRunnable extends Runnable
{
void doRun() throws Throwable;
default void run()
{
try
{
doRun();
}
catch (Throwable t)
{
Throwables.throwIfUnchecked(t);
throw new RuntimeException(t);
}
}
}