mirror of https://github.com/apache/cassandra
Adds a trie-based memtable implementation
patch by Branimir Lambov; reviewed by Jason Rutherglen, Jacek Lewandowski, Andres de la Peña and Caleb Rackliffe for CASSANDRA-17240
This commit is contained in:
parent
562cb26010
commit
7c55c73825
|
|
@ -127,6 +127,7 @@ default_env_vars: &default_env_vars
|
|||
# REPEATED_ANT_TEST_TARGET: test-jvm-dtest-some
|
||||
# REPEATED_ANT_TEST_TARGET: test-cdc
|
||||
# REPEATED_ANT_TEST_TARGET: test-compression
|
||||
# REPEATED_ANT_TEST_TARGET: test-trie
|
||||
# REPEATED_ANT_TEST_TARGET: test-system-keyspace-directory
|
||||
REPEATED_ANT_TEST_TARGET: testsome
|
||||
# The name of JUnit class to be run multiple times, for example:
|
||||
|
|
@ -323,6 +324,18 @@ j8_separate_jobs: &j8_separate_jobs
|
|||
requires:
|
||||
- start_utests_compression_repeat
|
||||
- j8_build
|
||||
- start_utests_trie:
|
||||
type: approval
|
||||
- utests_trie:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- start_utests_trie_repeat:
|
||||
type: approval
|
||||
- utests_trie_repeat:
|
||||
requires:
|
||||
- start_utests_trie_repeat
|
||||
- j8_build
|
||||
- start_utests_stress:
|
||||
type: approval
|
||||
- utests_stress:
|
||||
|
|
@ -559,6 +572,16 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs
|
|||
requires:
|
||||
- start_utests_compression
|
||||
- j8_build
|
||||
- start_utests_trie:
|
||||
type: approval
|
||||
- utests_trie:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- utests_trie_repeat:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- start_utests_stress:
|
||||
type: approval
|
||||
- utests_stress:
|
||||
|
|
@ -1018,6 +1041,16 @@ jobs:
|
|||
- run_parallel_junit_tests:
|
||||
target: testclasslist-compression
|
||||
|
||||
utests_trie:
|
||||
<<: *j8_par_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- create_junit_containers
|
||||
- log_environment
|
||||
- run_parallel_junit_tests:
|
||||
target: testclasslist-trie
|
||||
|
||||
utests_stress:
|
||||
<<: *j8_seq_executor
|
||||
steps:
|
||||
|
|
@ -1264,6 +1297,14 @@ jobs:
|
|||
- log_environment
|
||||
- run_utests_compression_repeat
|
||||
|
||||
utests_trie_repeat:
|
||||
<<: *j8_repeated_utest_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- log_environment
|
||||
- run_utests_trie_repeat
|
||||
|
||||
utests_system_keyspace_directory_repeat:
|
||||
<<: *j8_repeated_utest_executor
|
||||
steps:
|
||||
|
|
@ -1843,6 +1884,14 @@ commands:
|
|||
count: ${REPEATED_UTESTS_COUNT}
|
||||
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
|
||||
|
||||
run_utests_trie_repeat:
|
||||
steps:
|
||||
- run_repeated_utests:
|
||||
target: test-trie
|
||||
tests: ${REPEATED_UTESTS}
|
||||
count: ${REPEATED_UTESTS_COUNT}
|
||||
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
|
||||
|
||||
run_utests_system_keyspace_directory_repeat:
|
||||
steps:
|
||||
- run_repeated_utests:
|
||||
|
|
@ -1971,6 +2020,7 @@ commands:
|
|||
if [[ $target == "test" || \
|
||||
$target == "test-cdc" || \
|
||||
$target == "test-compression" || \
|
||||
$target == "test-trie" || \
|
||||
$target == "test-system-keyspace-directory" || \
|
||||
$target == "fqltool-test" || \
|
||||
$target == "long-test" || \
|
||||
|
|
@ -2097,6 +2147,7 @@ commands:
|
|||
if [[ $target == "test" || \
|
||||
$target == "test-cdc" || \
|
||||
$target == "test-compression" || \
|
||||
$target == "test-trie" || \
|
||||
$target == "test-system-keyspace-directory" || \
|
||||
$target == "fqltool-test" || \
|
||||
$target == "long-test" || \
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
--- config-2_1.yml 2022-10-15 12:44:53.884924328 +0100
|
||||
+++ config-2_1.yml.HIGHRES 2022-10-15 12:45:37.970163268 +0100
|
||||
@@ -148,14 +148,14 @@
|
||||
--- config-2_1.yml 2022-10-19 18:01:31.517748550 +0300
|
||||
+++ config-2_1.yml.HIGHRES 2022-10-19 18:06:37.789192360 +0300
|
||||
@@ -149,14 +149,14 @@
|
||||
j8_par_executor: &j8_par_executor
|
||||
executor:
|
||||
name: java8-executor
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
|
||||
j8_small_executor: &j8_small_executor
|
||||
executor:
|
||||
@@ -166,62 +166,68 @@
|
||||
@@ -167,62 +167,68 @@
|
||||
j8_medium_par_executor: &j8_medium_par_executor
|
||||
executor:
|
||||
name: java8-executor
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
--- config-2_1.yml 2022-10-15 12:44:53.884924328 +0100
|
||||
+++ config-2_1.yml.MIDRES 2022-10-15 12:45:37.963422950 +0100
|
||||
@@ -148,14 +148,14 @@
|
||||
--- config-2_1.yml 2022-10-19 18:01:31.517748550 +0300
|
||||
+++ config-2_1.yml.MIDRES 2022-10-19 18:06:26.777068521 +0300
|
||||
@@ -149,14 +149,14 @@
|
||||
j8_par_executor: &j8_par_executor
|
||||
executor:
|
||||
name: java8-executor
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
|
||||
j8_small_executor: &j8_small_executor
|
||||
executor:
|
||||
@@ -163,29 +163,41 @@
|
||||
@@ -164,29 +164,41 @@
|
||||
exec_resource_class: medium
|
||||
parallelism: 1
|
||||
|
||||
|
|
@ -68,7 +68,7 @@
|
|||
|
||||
j11_small_executor: &j11_small_executor
|
||||
executor:
|
||||
@@ -193,35 +205,47 @@
|
||||
@@ -194,35 +206,47 @@
|
||||
#exec_resource_class: medium
|
||||
parallelism: 1
|
||||
|
||||
|
|
@ -122,7 +122,7 @@
|
|||
|
||||
j8_separate_jobs: &j8_separate_jobs
|
||||
jobs:
|
||||
@@ -1045,7 +1069,7 @@
|
||||
@@ -1078,7 +1102,7 @@
|
||||
target: testclasslist-system-keyspace-directory
|
||||
|
||||
j8_dtests_vnode:
|
||||
|
|
@ -131,7 +131,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1059,7 +1083,7 @@
|
||||
@@ -1092,7 +1116,7 @@
|
||||
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
|
||||
|
||||
j11_dtests_vnode:
|
||||
|
|
@ -140,7 +140,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1074,7 +1098,7 @@
|
||||
@@ -1107,7 +1131,7 @@
|
||||
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
|
||||
|
||||
j8_dtests:
|
||||
|
|
@ -149,7 +149,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1088,7 +1112,7 @@
|
||||
@@ -1121,7 +1145,7 @@
|
||||
pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
|
||||
j11_dtests:
|
||||
|
|
@ -158,7 +158,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1103,7 +1127,7 @@
|
||||
@@ -1136,7 +1160,7 @@
|
||||
pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
|
||||
j8_upgrade_dtests:
|
||||
|
|
@ -167,7 +167,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1117,7 +1141,7 @@
|
||||
@@ -1150,7 +1174,7 @@
|
||||
pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
|
||||
|
||||
j8_cqlsh_dtests_py3_vnode:
|
||||
|
|
@ -176,7 +176,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1132,7 +1156,7 @@
|
||||
@@ -1165,7 +1189,7 @@
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
|
||||
|
||||
j8_cqlsh_dtests_py38_vnode:
|
||||
|
|
@ -185,7 +185,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1150,7 +1174,7 @@
|
||||
@@ -1183,7 +1207,7 @@
|
||||
python_version: '3.8'
|
||||
|
||||
j8_cqlsh_dtests_py3:
|
||||
|
|
@ -194,7 +194,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1165,7 +1189,7 @@
|
||||
@@ -1198,7 +1222,7 @@
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
|
||||
|
||||
j8_cqlsh_dtests_py38:
|
||||
|
|
@ -203,7 +203,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1183,7 +1207,7 @@
|
||||
@@ -1216,7 +1240,7 @@
|
||||
python_version: '3.8'
|
||||
|
||||
j11_cqlsh_dtests_py3_vnode:
|
||||
|
|
@ -212,7 +212,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1198,7 +1222,7 @@
|
||||
@@ -1231,7 +1255,7 @@
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
|
||||
|
||||
j11_cqlsh_dtests_py38_vnode:
|
||||
|
|
@ -221,7 +221,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1216,7 +1240,7 @@
|
||||
@@ -1249,7 +1273,7 @@
|
||||
python_version: '3.8'
|
||||
|
||||
j11_cqlsh_dtests_py3:
|
||||
|
|
@ -230,7 +230,7 @@
|
|||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
@@ -1231,7 +1255,7 @@
|
||||
@@ -1264,7 +1288,7 @@
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
|
||||
|
||||
j11_cqlsh_dtests_py38:
|
||||
|
|
|
|||
|
|
@ -170,7 +170,96 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/stdout
|
||||
destination: stdout
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: trunk
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
||||
- REPEATED_UTESTS: null
|
||||
- REPEATED_UTESTS_COUNT: 500
|
||||
- REPEATED_UTESTS_FQLTOOL: null
|
||||
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
||||
- REPEATED_UTESTS_LONG: null
|
||||
- REPEATED_UTESTS_LONG_COUNT: 100
|
||||
- REPEATED_UTESTS_STRESS: null
|
||||
- REPEATED_UTESTS_STRESS_COUNT: 500
|
||||
- REPEATED_SIMULATOR_DTESTS: null
|
||||
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_DTESTS: null
|
||||
- REPEATED_JVM_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_UPGRADE_DTESTS: null
|
||||
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
||||
- REPEATED_DTESTS: null
|
||||
- REPEATED_DTESTS_COUNT: 500
|
||||
- REPEATED_UPGRADE_DTESTS: null
|
||||
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
||||
- REPEATED_ANT_TEST_TARGET: testsome
|
||||
- REPEATED_ANT_TEST_CLASS: null
|
||||
- REPEATED_ANT_TEST_METHODS: null
|
||||
- REPEATED_ANT_TEST_VNODES: false
|
||||
- REPEATED_ANT_TEST_COUNT: 500
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_trie_repeat:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Log Environment Information
|
||||
command: |
|
||||
echo '*** id ***'
|
||||
id
|
||||
echo '*** cat /proc/cpuinfo ***'
|
||||
cat /proc/cpuinfo
|
||||
echo '*** free -m ***'
|
||||
free -m
|
||||
echo '*** df -m ***'
|
||||
df -m
|
||||
echo '*** ifconfig -a ***'
|
||||
ifconfig -a
|
||||
echo '*** uname -a ***'
|
||||
uname -a
|
||||
echo '*** mount ***'
|
||||
mount
|
||||
echo '*** env ***'
|
||||
env
|
||||
echo '*** java ***'
|
||||
which java
|
||||
java -version
|
||||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -414,7 +503,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -983,6 +1072,7 @@ jobs:
|
|||
if [[ $target == "test" || \
|
||||
$target == "test-cdc" || \
|
||||
$target == "test-compression" || \
|
||||
$target == "test-trie" || \
|
||||
$target == "test-system-keyspace-directory" || \
|
||||
$target == "fqltool-test" || \
|
||||
$target == "long-test" || \
|
||||
|
|
@ -1577,7 +1667,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -1666,7 +1756,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -1887,7 +1977,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2131,7 +2221,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2323,7 +2413,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2648,7 +2738,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2821,7 +2911,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3014,7 +3104,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3410,7 +3500,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3838,7 +3928,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -4072,6 +4162,7 @@ jobs:
|
|||
if [[ $target == "test" || \
|
||||
$target == "test-cdc" || \
|
||||
$target == "test-compression" || \
|
||||
$target == "test-trie" || \
|
||||
$target == "test-system-keyspace-directory" || \
|
||||
$target == "fqltool-test" || \
|
||||
$target == "long-test" || \
|
||||
|
|
@ -4756,7 +4847,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -4811,6 +4902,123 @@ jobs:
|
|||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
utests_trie:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Determine unit Tests to Run
|
||||
command: |
|
||||
# reminder: this code (along with all the steps) is independently executed on every circle container
|
||||
# so the goal here is to get the circleci script to return the tests *this* container will run
|
||||
# which we do via the `circleci` cli tool.
|
||||
|
||||
rm -fr ~/cassandra-dtest/upgrade_tests
|
||||
echo "***java tests***"
|
||||
|
||||
# get all of our unit test filenames
|
||||
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
||||
|
||||
# split up the unit tests into groups based on the number of containers we have
|
||||
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
||||
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
||||
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
no_output_timeout: 15m
|
||||
- run:
|
||||
name: Log Environment Information
|
||||
command: |
|
||||
echo '*** id ***'
|
||||
id
|
||||
echo '*** cat /proc/cpuinfo ***'
|
||||
cat /proc/cpuinfo
|
||||
echo '*** free -m ***'
|
||||
free -m
|
||||
echo '*** df -m ***'
|
||||
df -m
|
||||
echo '*** ifconfig -a ***'
|
||||
ifconfig -a
|
||||
echo '*** uname -a ***'
|
||||
uname -a
|
||||
echo '*** mount ***'
|
||||
mount
|
||||
echo '*** env ***'
|
||||
env
|
||||
echo '*** java ***'
|
||||
which java
|
||||
java -version
|
||||
- run:
|
||||
name: Run Unit Tests (testclasslist-trie)
|
||||
command: |
|
||||
set -x
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
||||
if [ -z "$test_timeout" ]; then
|
||||
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
||||
fi
|
||||
ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
path: /tmp/cassandra/build/test/output/
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: trunk
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
||||
- REPEATED_UTESTS: null
|
||||
- REPEATED_UTESTS_COUNT: 500
|
||||
- REPEATED_UTESTS_FQLTOOL: null
|
||||
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
||||
- REPEATED_UTESTS_LONG: null
|
||||
- REPEATED_UTESTS_LONG_COUNT: 100
|
||||
- REPEATED_UTESTS_STRESS: null
|
||||
- REPEATED_UTESTS_STRESS_COUNT: 500
|
||||
- REPEATED_SIMULATOR_DTESTS: null
|
||||
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_DTESTS: null
|
||||
- REPEATED_JVM_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_UPGRADE_DTESTS: null
|
||||
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
||||
- REPEATED_DTESTS: null
|
||||
- REPEATED_DTESTS_COUNT: 500
|
||||
- REPEATED_UPGRADE_DTESTS: null
|
||||
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
||||
- REPEATED_ANT_TEST_TARGET: testsome
|
||||
- REPEATED_ANT_TEST_CLASS: null
|
||||
- REPEATED_ANT_TEST_METHODS: null
|
||||
- REPEATED_ANT_TEST_VNODES: false
|
||||
- REPEATED_ANT_TEST_COUNT: 500
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_system_keyspace_directory:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
|
|
@ -5445,6 +5653,12 @@ workflows:
|
|||
requires:
|
||||
- start_utests_compression
|
||||
- j8_build
|
||||
- start_utests_trie:
|
||||
type: approval
|
||||
- utests_trie:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- start_utests_stress:
|
||||
type: approval
|
||||
- utests_stress:
|
||||
|
|
@ -5578,6 +5792,12 @@ workflows:
|
|||
requires:
|
||||
- start_utests_compression
|
||||
- j8_build
|
||||
- start_utests_trie:
|
||||
type: approval
|
||||
- utests_trie:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- start_utests_stress:
|
||||
type: approval
|
||||
- utests_stress:
|
||||
|
|
|
|||
|
|
@ -170,7 +170,96 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/stdout
|
||||
destination: stdout
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: trunk
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
||||
- REPEATED_UTESTS: null
|
||||
- REPEATED_UTESTS_COUNT: 500
|
||||
- REPEATED_UTESTS_FQLTOOL: null
|
||||
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
||||
- REPEATED_UTESTS_LONG: null
|
||||
- REPEATED_UTESTS_LONG_COUNT: 100
|
||||
- REPEATED_UTESTS_STRESS: null
|
||||
- REPEATED_UTESTS_STRESS_COUNT: 500
|
||||
- REPEATED_SIMULATOR_DTESTS: null
|
||||
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_DTESTS: null
|
||||
- REPEATED_JVM_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_UPGRADE_DTESTS: null
|
||||
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
||||
- REPEATED_DTESTS: null
|
||||
- REPEATED_DTESTS_COUNT: 500
|
||||
- REPEATED_UPGRADE_DTESTS: null
|
||||
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
||||
- REPEATED_ANT_TEST_TARGET: testsome
|
||||
- REPEATED_ANT_TEST_CLASS: null
|
||||
- REPEATED_ANT_TEST_METHODS: null
|
||||
- REPEATED_ANT_TEST_VNODES: false
|
||||
- REPEATED_ANT_TEST_COUNT: 500
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_trie_repeat:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 100
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Log Environment Information
|
||||
command: |
|
||||
echo '*** id ***'
|
||||
id
|
||||
echo '*** cat /proc/cpuinfo ***'
|
||||
cat /proc/cpuinfo
|
||||
echo '*** free -m ***'
|
||||
free -m
|
||||
echo '*** df -m ***'
|
||||
df -m
|
||||
echo '*** ifconfig -a ***'
|
||||
ifconfig -a
|
||||
echo '*** uname -a ***'
|
||||
uname -a
|
||||
echo '*** mount ***'
|
||||
mount
|
||||
echo '*** env ***'
|
||||
env
|
||||
echo '*** java ***'
|
||||
which java
|
||||
java -version
|
||||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -414,7 +503,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -983,6 +1072,7 @@ jobs:
|
|||
if [[ $target == "test" || \
|
||||
$target == "test-cdc" || \
|
||||
$target == "test-compression" || \
|
||||
$target == "test-trie" || \
|
||||
$target == "test-system-keyspace-directory" || \
|
||||
$target == "fqltool-test" || \
|
||||
$target == "long-test" || \
|
||||
|
|
@ -1577,7 +1667,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -1666,7 +1756,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -1887,7 +1977,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2131,7 +2221,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2323,7 +2413,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2648,7 +2738,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2821,7 +2911,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3014,7 +3104,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3410,7 +3500,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3838,7 +3928,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -4072,6 +4162,7 @@ jobs:
|
|||
if [[ $target == "test" || \
|
||||
$target == "test-cdc" || \
|
||||
$target == "test-compression" || \
|
||||
$target == "test-trie" || \
|
||||
$target == "test-system-keyspace-directory" || \
|
||||
$target == "fqltool-test" || \
|
||||
$target == "long-test" || \
|
||||
|
|
@ -4756,7 +4847,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -4811,6 +4902,123 @@ jobs:
|
|||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
utests_trie:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 100
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Determine unit Tests to Run
|
||||
command: |
|
||||
# reminder: this code (along with all the steps) is independently executed on every circle container
|
||||
# so the goal here is to get the circleci script to return the tests *this* container will run
|
||||
# which we do via the `circleci` cli tool.
|
||||
|
||||
rm -fr ~/cassandra-dtest/upgrade_tests
|
||||
echo "***java tests***"
|
||||
|
||||
# get all of our unit test filenames
|
||||
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
||||
|
||||
# split up the unit tests into groups based on the number of containers we have
|
||||
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
||||
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
||||
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
no_output_timeout: 15m
|
||||
- run:
|
||||
name: Log Environment Information
|
||||
command: |
|
||||
echo '*** id ***'
|
||||
id
|
||||
echo '*** cat /proc/cpuinfo ***'
|
||||
cat /proc/cpuinfo
|
||||
echo '*** free -m ***'
|
||||
free -m
|
||||
echo '*** df -m ***'
|
||||
df -m
|
||||
echo '*** ifconfig -a ***'
|
||||
ifconfig -a
|
||||
echo '*** uname -a ***'
|
||||
uname -a
|
||||
echo '*** mount ***'
|
||||
mount
|
||||
echo '*** env ***'
|
||||
env
|
||||
echo '*** java ***'
|
||||
which java
|
||||
java -version
|
||||
- run:
|
||||
name: Run Unit Tests (testclasslist-trie)
|
||||
command: |
|
||||
set -x
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
||||
if [ -z "$test_timeout" ]; then
|
||||
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
||||
fi
|
||||
ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
path: /tmp/cassandra/build/test/output/
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: trunk
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
||||
- REPEATED_UTESTS: null
|
||||
- REPEATED_UTESTS_COUNT: 500
|
||||
- REPEATED_UTESTS_FQLTOOL: null
|
||||
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
||||
- REPEATED_UTESTS_LONG: null
|
||||
- REPEATED_UTESTS_LONG_COUNT: 100
|
||||
- REPEATED_UTESTS_STRESS: null
|
||||
- REPEATED_UTESTS_STRESS_COUNT: 500
|
||||
- REPEATED_SIMULATOR_DTESTS: null
|
||||
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_DTESTS: null
|
||||
- REPEATED_JVM_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_UPGRADE_DTESTS: null
|
||||
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
||||
- REPEATED_DTESTS: null
|
||||
- REPEATED_DTESTS_COUNT: 500
|
||||
- REPEATED_UPGRADE_DTESTS: null
|
||||
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
||||
- REPEATED_ANT_TEST_TARGET: testsome
|
||||
- REPEATED_ANT_TEST_CLASS: null
|
||||
- REPEATED_ANT_TEST_METHODS: null
|
||||
- REPEATED_ANT_TEST_VNODES: false
|
||||
- REPEATED_ANT_TEST_COUNT: 500
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_system_keyspace_directory:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
|
|
@ -5487,6 +5695,18 @@ workflows:
|
|||
requires:
|
||||
- start_utests_compression_repeat
|
||||
- j8_build
|
||||
- start_utests_trie:
|
||||
type: approval
|
||||
- utests_trie:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- start_utests_trie_repeat:
|
||||
type: approval
|
||||
- utests_trie_repeat:
|
||||
requires:
|
||||
- start_utests_trie_repeat
|
||||
- j8_build
|
||||
- start_utests_stress:
|
||||
type: approval
|
||||
- utests_stress:
|
||||
|
|
@ -5709,6 +5929,16 @@ workflows:
|
|||
requires:
|
||||
- start_utests_compression
|
||||
- j8_build
|
||||
- start_utests_trie:
|
||||
type: approval
|
||||
- utests_trie:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- utests_trie_repeat:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- start_utests_stress:
|
||||
type: approval
|
||||
- utests_stress:
|
||||
|
|
|
|||
|
|
@ -170,7 +170,96 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/stdout
|
||||
destination: stdout
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: trunk
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
||||
- REPEATED_UTESTS: null
|
||||
- REPEATED_UTESTS_COUNT: 500
|
||||
- REPEATED_UTESTS_FQLTOOL: null
|
||||
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
||||
- REPEATED_UTESTS_LONG: null
|
||||
- REPEATED_UTESTS_LONG_COUNT: 100
|
||||
- REPEATED_UTESTS_STRESS: null
|
||||
- REPEATED_UTESTS_STRESS_COUNT: 500
|
||||
- REPEATED_SIMULATOR_DTESTS: null
|
||||
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_DTESTS: null
|
||||
- REPEATED_JVM_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_UPGRADE_DTESTS: null
|
||||
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
||||
- REPEATED_DTESTS: null
|
||||
- REPEATED_DTESTS_COUNT: 500
|
||||
- REPEATED_UPGRADE_DTESTS: null
|
||||
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
||||
- REPEATED_ANT_TEST_TARGET: testsome
|
||||
- REPEATED_ANT_TEST_CLASS: null
|
||||
- REPEATED_ANT_TEST_METHODS: null
|
||||
- REPEATED_ANT_TEST_VNODES: false
|
||||
- REPEATED_ANT_TEST_COUNT: 500
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_trie_repeat:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Log Environment Information
|
||||
command: |
|
||||
echo '*** id ***'
|
||||
id
|
||||
echo '*** cat /proc/cpuinfo ***'
|
||||
cat /proc/cpuinfo
|
||||
echo '*** free -m ***'
|
||||
free -m
|
||||
echo '*** df -m ***'
|
||||
df -m
|
||||
echo '*** ifconfig -a ***'
|
||||
ifconfig -a
|
||||
echo '*** uname -a ***'
|
||||
uname -a
|
||||
echo '*** mount ***'
|
||||
mount
|
||||
echo '*** env ***'
|
||||
env
|
||||
echo '*** java ***'
|
||||
which java
|
||||
java -version
|
||||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -414,7 +503,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -983,6 +1072,7 @@ jobs:
|
|||
if [[ $target == "test" || \
|
||||
$target == "test-cdc" || \
|
||||
$target == "test-compression" || \
|
||||
$target == "test-trie" || \
|
||||
$target == "test-system-keyspace-directory" || \
|
||||
$target == "fqltool-test" || \
|
||||
$target == "long-test" || \
|
||||
|
|
@ -1577,7 +1667,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -1666,7 +1756,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -1887,7 +1977,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2131,7 +2221,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2323,7 +2413,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2648,7 +2738,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2821,7 +2911,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3014,7 +3104,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3410,7 +3500,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3838,7 +3928,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -4072,6 +4162,7 @@ jobs:
|
|||
if [[ $target == "test" || \
|
||||
$target == "test-cdc" || \
|
||||
$target == "test-compression" || \
|
||||
$target == "test-trie" || \
|
||||
$target == "test-system-keyspace-directory" || \
|
||||
$target == "fqltool-test" || \
|
||||
$target == "long-test" || \
|
||||
|
|
@ -4756,7 +4847,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -4811,6 +4902,123 @@ jobs:
|
|||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
utests_trie:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Determine unit Tests to Run
|
||||
command: |
|
||||
# reminder: this code (along with all the steps) is independently executed on every circle container
|
||||
# so the goal here is to get the circleci script to return the tests *this* container will run
|
||||
# which we do via the `circleci` cli tool.
|
||||
|
||||
rm -fr ~/cassandra-dtest/upgrade_tests
|
||||
echo "***java tests***"
|
||||
|
||||
# get all of our unit test filenames
|
||||
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
||||
|
||||
# split up the unit tests into groups based on the number of containers we have
|
||||
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
||||
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
||||
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
no_output_timeout: 15m
|
||||
- run:
|
||||
name: Log Environment Information
|
||||
command: |
|
||||
echo '*** id ***'
|
||||
id
|
||||
echo '*** cat /proc/cpuinfo ***'
|
||||
cat /proc/cpuinfo
|
||||
echo '*** free -m ***'
|
||||
free -m
|
||||
echo '*** df -m ***'
|
||||
df -m
|
||||
echo '*** ifconfig -a ***'
|
||||
ifconfig -a
|
||||
echo '*** uname -a ***'
|
||||
uname -a
|
||||
echo '*** mount ***'
|
||||
mount
|
||||
echo '*** env ***'
|
||||
env
|
||||
echo '*** java ***'
|
||||
which java
|
||||
java -version
|
||||
- run:
|
||||
name: Run Unit Tests (testclasslist-trie)
|
||||
command: |
|
||||
set -x
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
||||
if [ -z "$test_timeout" ]; then
|
||||
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
||||
fi
|
||||
ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
path: /tmp/cassandra/build/test/output/
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: trunk
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
||||
- REPEATED_UTESTS: null
|
||||
- REPEATED_UTESTS_COUNT: 500
|
||||
- REPEATED_UTESTS_FQLTOOL: null
|
||||
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
||||
- REPEATED_UTESTS_LONG: null
|
||||
- REPEATED_UTESTS_LONG_COUNT: 100
|
||||
- REPEATED_UTESTS_STRESS: null
|
||||
- REPEATED_UTESTS_STRESS_COUNT: 500
|
||||
- REPEATED_SIMULATOR_DTESTS: null
|
||||
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_DTESTS: null
|
||||
- REPEATED_JVM_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_UPGRADE_DTESTS: null
|
||||
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
||||
- REPEATED_DTESTS: null
|
||||
- REPEATED_DTESTS_COUNT: 500
|
||||
- REPEATED_UPGRADE_DTESTS: null
|
||||
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
||||
- REPEATED_ANT_TEST_TARGET: testsome
|
||||
- REPEATED_ANT_TEST_CLASS: null
|
||||
- REPEATED_ANT_TEST_METHODS: null
|
||||
- REPEATED_ANT_TEST_VNODES: false
|
||||
- REPEATED_ANT_TEST_COUNT: 500
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_system_keyspace_directory:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
|
|
@ -5487,6 +5695,18 @@ workflows:
|
|||
requires:
|
||||
- start_utests_compression_repeat
|
||||
- j8_build
|
||||
- start_utests_trie:
|
||||
type: approval
|
||||
- utests_trie:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- start_utests_trie_repeat:
|
||||
type: approval
|
||||
- utests_trie_repeat:
|
||||
requires:
|
||||
- start_utests_trie_repeat
|
||||
- j8_build
|
||||
- start_utests_stress:
|
||||
type: approval
|
||||
- utests_stress:
|
||||
|
|
@ -5709,6 +5929,16 @@ workflows:
|
|||
requires:
|
||||
- start_utests_compression
|
||||
- j8_build
|
||||
- start_utests_trie:
|
||||
type: approval
|
||||
- utests_trie:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- utests_trie_repeat:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- start_utests_stress:
|
||||
type: approval
|
||||
- utests_stress:
|
||||
|
|
|
|||
|
|
@ -170,7 +170,96 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-compression $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/stdout
|
||||
destination: stdout
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/results/repeated_utests/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: trunk
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
||||
- REPEATED_UTESTS: null
|
||||
- REPEATED_UTESTS_COUNT: 500
|
||||
- REPEATED_UTESTS_FQLTOOL: null
|
||||
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
||||
- REPEATED_UTESTS_LONG: null
|
||||
- REPEATED_UTESTS_LONG_COUNT: 100
|
||||
- REPEATED_UTESTS_STRESS: null
|
||||
- REPEATED_UTESTS_STRESS_COUNT: 500
|
||||
- REPEATED_SIMULATOR_DTESTS: null
|
||||
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_DTESTS: null
|
||||
- REPEATED_JVM_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_UPGRADE_DTESTS: null
|
||||
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
||||
- REPEATED_DTESTS: null
|
||||
- REPEATED_DTESTS_COUNT: 500
|
||||
- REPEATED_UPGRADE_DTESTS: null
|
||||
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
||||
- REPEATED_ANT_TEST_TARGET: testsome
|
||||
- REPEATED_ANT_TEST_CLASS: null
|
||||
- REPEATED_ANT_TEST_METHODS: null
|
||||
- REPEATED_ANT_TEST_VNODES: false
|
||||
- REPEATED_ANT_TEST_COUNT: 500
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_trie_repeat:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 25
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Log Environment Information
|
||||
command: |
|
||||
echo '*** id ***'
|
||||
id
|
||||
echo '*** cat /proc/cpuinfo ***'
|
||||
cat /proc/cpuinfo
|
||||
echo '*** free -m ***'
|
||||
free -m
|
||||
echo '*** df -m ***'
|
||||
df -m
|
||||
echo '*** ifconfig -a ***'
|
||||
ifconfig -a
|
||||
echo '*** uname -a ***'
|
||||
uname -a
|
||||
echo '*** mount ***'
|
||||
mount
|
||||
echo '*** env ***'
|
||||
env
|
||||
echo '*** java ***'
|
||||
which java
|
||||
java -version
|
||||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-trie $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -414,7 +503,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant stress-test-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -983,6 +1072,7 @@ jobs:
|
|||
if [[ $target == "test" || \
|
||||
$target == "test-cdc" || \
|
||||
$target == "test-compression" || \
|
||||
$target == "test-trie" || \
|
||||
$target == "test-system-keyspace-directory" || \
|
||||
$target == "fqltool-test" || \
|
||||
$target == "long-test" || \
|
||||
|
|
@ -1577,7 +1667,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_LONG} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant long-testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -1666,7 +1756,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -1887,7 +1977,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-system-keyspace-directory $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2131,7 +2221,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2323,7 +2413,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_SIMULATOR_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-simulator-dtest $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2648,7 +2738,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant fqltool-test $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -2821,7 +2911,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3014,7 +3104,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3410,7 +3500,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=true\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -3838,7 +3928,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_DTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -4072,6 +4162,7 @@ jobs:
|
|||
if [[ $target == "test" || \
|
||||
$target == "test-cdc" || \
|
||||
$target == "test-compression" || \
|
||||
$target == "test-trie" || \
|
||||
$target == "test-system-keyspace-directory" || \
|
||||
$target == "fqltool-test" || \
|
||||
$target == "long-test" || \
|
||||
|
|
@ -4756,7 +4847,7 @@ jobs:
|
|||
- run:
|
||||
name: Repeatedly run new or modifed JUnit tests
|
||||
no_output_timeout: 15m
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_UTESTS} | sed -e \"s/<nil>//\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [ $method == \"\" ]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant testsome $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results/repeated_utests/output
|
||||
- store_artifacts:
|
||||
|
|
@ -4811,6 +4902,123 @@ jobs:
|
|||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
utests_trie:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 25
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Determine unit Tests to Run
|
||||
command: |
|
||||
# reminder: this code (along with all the steps) is independently executed on every circle container
|
||||
# so the goal here is to get the circleci script to return the tests *this* container will run
|
||||
# which we do via the `circleci` cli tool.
|
||||
|
||||
rm -fr ~/cassandra-dtest/upgrade_tests
|
||||
echo "***java tests***"
|
||||
|
||||
# get all of our unit test filenames
|
||||
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
|
||||
|
||||
# split up the unit tests into groups based on the number of containers we have
|
||||
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
||||
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
|
||||
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
no_output_timeout: 15m
|
||||
- run:
|
||||
name: Log Environment Information
|
||||
command: |
|
||||
echo '*** id ***'
|
||||
id
|
||||
echo '*** cat /proc/cpuinfo ***'
|
||||
cat /proc/cpuinfo
|
||||
echo '*** free -m ***'
|
||||
free -m
|
||||
echo '*** df -m ***'
|
||||
df -m
|
||||
echo '*** ifconfig -a ***'
|
||||
ifconfig -a
|
||||
echo '*** uname -a ***'
|
||||
uname -a
|
||||
echo '*** mount ***'
|
||||
mount
|
||||
echo '*** env ***'
|
||||
env
|
||||
echo '*** java ***'
|
||||
which java
|
||||
java -version
|
||||
- run:
|
||||
name: Run Unit Tests (testclasslist-trie)
|
||||
command: |
|
||||
set -x
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
|
||||
if [ -z "$test_timeout" ]; then
|
||||
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
|
||||
fi
|
||||
ant testclasslist-trie -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
path: /tmp/cassandra/build/test/output/
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: trunk
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- REPEATED_TESTS_STOP_ON_FAILURE: false
|
||||
- REPEATED_UTESTS: null
|
||||
- REPEATED_UTESTS_COUNT: 500
|
||||
- REPEATED_UTESTS_FQLTOOL: null
|
||||
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
|
||||
- REPEATED_UTESTS_LONG: null
|
||||
- REPEATED_UTESTS_LONG_COUNT: 100
|
||||
- REPEATED_UTESTS_STRESS: null
|
||||
- REPEATED_UTESTS_STRESS_COUNT: 500
|
||||
- REPEATED_SIMULATOR_DTESTS: null
|
||||
- REPEATED_SIMULATOR_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_DTESTS: null
|
||||
- REPEATED_JVM_DTESTS_COUNT: 500
|
||||
- REPEATED_JVM_UPGRADE_DTESTS: null
|
||||
- REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
|
||||
- REPEATED_DTESTS: null
|
||||
- REPEATED_DTESTS_COUNT: 500
|
||||
- REPEATED_UPGRADE_DTESTS: null
|
||||
- REPEATED_UPGRADE_DTESTS_COUNT: 25
|
||||
- REPEATED_ANT_TEST_TARGET: testsome
|
||||
- REPEATED_ANT_TEST_CLASS: null
|
||||
- REPEATED_ANT_TEST_METHODS: null
|
||||
- REPEATED_ANT_TEST_VNODES: false
|
||||
- REPEATED_ANT_TEST_COUNT: 500
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_system_keyspace_directory:
|
||||
docker:
|
||||
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
|
||||
|
|
@ -5487,6 +5695,18 @@ workflows:
|
|||
requires:
|
||||
- start_utests_compression_repeat
|
||||
- j8_build
|
||||
- start_utests_trie:
|
||||
type: approval
|
||||
- utests_trie:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- start_utests_trie_repeat:
|
||||
type: approval
|
||||
- utests_trie_repeat:
|
||||
requires:
|
||||
- start_utests_trie_repeat
|
||||
- j8_build
|
||||
- start_utests_stress:
|
||||
type: approval
|
||||
- utests_stress:
|
||||
|
|
@ -5709,6 +5929,16 @@ workflows:
|
|||
requires:
|
||||
- start_utests_compression
|
||||
- j8_build
|
||||
- start_utests_trie:
|
||||
type: approval
|
||||
- utests_trie:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- utests_trie_repeat:
|
||||
requires:
|
||||
- start_utests_trie
|
||||
- j8_build
|
||||
- start_utests_stress:
|
||||
type: approval
|
||||
- utests_stress:
|
||||
|
|
|
|||
|
|
@ -259,6 +259,7 @@ if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS=" )); then
|
|||
delete_job "j8_unit_tests_repeat"
|
||||
delete_job "j11_unit_tests_repeat"
|
||||
delete_job "utests_compression_repeat"
|
||||
delete_job "utests_trie_repeat"
|
||||
delete_job "utests_system_keyspace_directory_repeat"
|
||||
fi
|
||||
if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS_LONG=")); then
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
4.2
|
||||
* Adds a trie-based memtable implementation (CASSANDRA-17240)
|
||||
* Further improves precision of memtable heap tracking (CASSANDRA-17240)
|
||||
* Fix formatting of metrics documentation (CASSANDRA-17961)
|
||||
* Keep sstable level when streaming for decommission and move (CASSANDRA-17969)
|
||||
* Add Unavailables metric for CASWrite in the docs (CASSANDRA-16357)
|
||||
|
|
|
|||
3
NEWS.txt
3
NEWS.txt
|
|
@ -57,6 +57,9 @@ using the provided 'sstableupgrade' tool.
|
|||
|
||||
New features
|
||||
------------
|
||||
- Adds a trie-based memtable implementation, which improves memory use, garbage collection efficiency and lookup
|
||||
performance. The new memtable is implemented by the TrieMemtable class and can be selected using the memtable
|
||||
API, see src/java/org/apache/cassandra/db/memtable/Memtable_API.md.
|
||||
- Added a new configuration cdc_on_repair_enabled to toggle whether CDC mutations are replayed through the
|
||||
write path on streaming, e.g. repair. When enabled, CDC data streamed to the destination node will be written into
|
||||
commit log first. When disabled, the streamed CDC data is written into SSTables just the same as normal streaming.
|
||||
|
|
|
|||
37
build.xml
37
build.xml
|
|
@ -1190,7 +1190,27 @@
|
|||
</sequential>
|
||||
</macrodef>
|
||||
|
||||
<macrodef name="testlist-system-keyspace-directory">
|
||||
<macrodef name="testlist-trie">
|
||||
<attribute name="test.file.list" />
|
||||
<sequential>
|
||||
<property name="trie_yaml" value="${build.test.dir}/cassandra.trie.yaml"/>
|
||||
<concat destfile="${trie_yaml}">
|
||||
<fileset file="${test.conf}/cassandra.yaml"/>
|
||||
<fileset file="${test.conf}/trie_memtable.yaml"/>
|
||||
</concat>
|
||||
<testmacrohelper inputdir="${test.unit.src}" filelist="@{test.file.list}"
|
||||
exclude="**/*.java" timeout="${test.timeout}" testtag="trie">
|
||||
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
|
||||
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
|
||||
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
|
||||
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
|
||||
<jvmarg value="-Dcassandra.config=file:///${trie_yaml}"/>
|
||||
<jvmarg value="-Dcassandra.skip_sync=true" />
|
||||
</testmacrohelper>
|
||||
</sequential>
|
||||
</macrodef>
|
||||
|
||||
<macrodef name="testlist-system-keyspace-directory">
|
||||
<attribute name="test.file.list" />
|
||||
<sequential>
|
||||
<property name="system_keyspaces_directory_yaml" value="${build.test.dir}/cassandra.system.yaml"/>
|
||||
|
|
@ -1298,6 +1318,14 @@
|
|||
<testhelper testdelegate="testlist-cdc" />
|
||||
</target>
|
||||
|
||||
<target name="test-trie" depends="build-test" description="Execute unit tests with trie memtables">
|
||||
<path id="all-test-classes-path">
|
||||
<fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
|
||||
</path>
|
||||
<property name="all-test-classes" refid="all-test-classes-path"/>
|
||||
<testhelper testdelegate="testlist-trie" />
|
||||
</target>
|
||||
|
||||
<target name="test-system-keyspace-directory" depends="build-test" description="Execute unit tests with a system keyspaces directory configured">
|
||||
<path id="all-test-classes-path">
|
||||
<fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
|
||||
|
|
@ -1545,6 +1573,13 @@
|
|||
<property name="all-test-classes" refid="all-test-classes-path"/>
|
||||
<testhelper testdelegate="testlist-compression"/>
|
||||
</target>
|
||||
<target name="testclasslist-trie" depends="build-test" description="Run tests given in file -Dtest.classlistfile (one-class-per-line, e.g. org/apache/cassandra/db/SomeTest.java)">
|
||||
<path id="all-test-classes-path">
|
||||
<fileset dir="${test.dir}/${test.classlistprefix}" includesfile="${test.classlistfile}"/>
|
||||
</path>
|
||||
<property name="all-test-classes" refid="all-test-classes-path"/>
|
||||
<testhelper testdelegate="testlist-trie"/>
|
||||
</target>
|
||||
<target name="testclasslist-cdc" depends="build-test" description="Run tests given in file -Dtest.classlistfile (one-class-per-line, e.g. org/apache/cassandra/db/SomeTest.java)">
|
||||
<path id="all-test-classes-path">
|
||||
<fileset dir="${test.dir}/${test.classlistprefix}" includesfile="${test.classlistfile}"/>
|
||||
|
|
|
|||
|
|
@ -49,7 +49,8 @@ public abstract class AbstractArrayClusteringPrefix extends AbstractOnHeapCluste
|
|||
return out;
|
||||
}
|
||||
|
||||
public ClusteringPrefix<byte[]> minimize()
|
||||
@Override
|
||||
public ClusteringPrefix<byte[]> retainable()
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,10 +42,19 @@ public abstract class AbstractBufferClusteringPrefix extends AbstractOnHeapClust
|
|||
return getRawValues();
|
||||
}
|
||||
|
||||
public ClusteringPrefix<ByteBuffer> minimize()
|
||||
@Override
|
||||
public ClusteringPrefix<ByteBuffer> retainable()
|
||||
{
|
||||
if (!ByteBufferUtil.canMinimize(values))
|
||||
return this;
|
||||
return new BufferClustering(ByteBufferUtil.minimizeBuffers(values));
|
||||
|
||||
ByteBuffer[] minimizedValues = ByteBufferUtil.minimizeBuffers(this.values);
|
||||
if (kind.isBoundary())
|
||||
return accessor().factory().boundary(kind, minimizedValues);
|
||||
if (kind.isBound())
|
||||
return accessor().factory().bound(kind, minimizedValues);
|
||||
|
||||
assert kind() != Kind.STATIC_CLUSTERING; // not minimizable
|
||||
return accessor().factory().clustering(minimizedValues);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,12 @@ public class BufferDecoratedKey extends DecoratedKey
|
|||
return key;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getKeyLength()
|
||||
{
|
||||
return key.remaining();
|
||||
}
|
||||
|
||||
/**
|
||||
* A factory method that translates the given byte-comparable representation to a {@link BufferDecoratedKey}
|
||||
* instance. If the given byte comparable doesn't represent the encoding of a buffer decorated key, anything from a
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ import org.apache.cassandra.io.util.DataInputPlus;
|
|||
import org.apache.cassandra.io.util.DataOutputPlus;
|
||||
import org.apache.cassandra.schema.TableMetadata;
|
||||
import org.apache.cassandra.utils.ByteArrayUtil;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
import org.apache.cassandra.utils.bytecomparable.ByteComparable.Version;
|
||||
import org.apache.cassandra.utils.bytecomparable.ByteSource;
|
||||
|
||||
|
|
@ -361,10 +360,11 @@ public interface ClusteringPrefix<V> extends IMeasurableMemory, Clusterable<V>
|
|||
public ByteBuffer[] getBufferArray();
|
||||
|
||||
/**
|
||||
* If the prefix contains byte buffers that can be minimized (see {@link ByteBufferUtil#minimalBufferFor(ByteBuffer)}),
|
||||
* this will return a copy of the prefix with minimized values, otherwise it returns itself.
|
||||
* Return the key in a form that can be retained for longer-term use. This means extracting keys stored in shared
|
||||
* memory (i.e. in memtables) to minimized on-heap versions.
|
||||
* If the object is already in minimal form, no action will be taken.
|
||||
*/
|
||||
public ClusteringPrefix<V> minimize();
|
||||
public ClusteringPrefix<V> retainable();
|
||||
|
||||
public static class Serializer
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1451,41 +1451,63 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
|
|||
@Override
|
||||
public ShardBoundaries localRangeSplits(int shardCount)
|
||||
{
|
||||
if (shardCount == 1 || !getPartitioner().splitter().isPresent() || SchemaConstants.isLocalSystemKeyspace(keyspace.getName()))
|
||||
if (shardCount == 1 || !getPartitioner().splitter().isPresent())
|
||||
return ShardBoundaries.NONE;
|
||||
|
||||
ShardBoundaries shardBoundaries = cachedShardBoundaries;
|
||||
|
||||
if (shardBoundaries == null ||
|
||||
shardBoundaries.shardCount() != shardCount ||
|
||||
shardBoundaries.ringVersion != StorageService.instance.getTokenMetadata().getRingVersion())
|
||||
shardBoundaries.ringVersion != -1 && shardBoundaries.ringVersion != StorageService.instance.getTokenMetadata().getRingVersion())
|
||||
{
|
||||
DiskBoundaryManager.VersionedRangesAtEndpoint versionedLocalRanges = DiskBoundaryManager.getVersionedLocalRanges(this);
|
||||
Set<Range<Token>> localRanges = versionedLocalRanges.rangesAtEndpoint.ranges();
|
||||
List<Splitter.WeightedRange> weightedRanges;
|
||||
if (localRanges.isEmpty())
|
||||
weightedRanges = ImmutableList.of(new Splitter.WeightedRange(1.0, new Range<>(getPartitioner().getMinimumToken(), getPartitioner().getMaximumToken())));
|
||||
long ringVersion;
|
||||
if (!SchemaConstants.isLocalSystemKeyspace(keyspace.getName())
|
||||
&& getPartitioner() == StorageService.instance.getTokenMetadata().partitioner)
|
||||
{
|
||||
DiskBoundaryManager.VersionedRangesAtEndpoint versionedLocalRanges = DiskBoundaryManager.getVersionedLocalRanges(this);
|
||||
Set<Range<Token>> localRanges = versionedLocalRanges.rangesAtEndpoint.ranges();
|
||||
ringVersion = versionedLocalRanges.ringVersion;
|
||||
|
||||
if (!localRanges.isEmpty())
|
||||
{
|
||||
weightedRanges = new ArrayList<>(localRanges.size());
|
||||
for (Range<Token> r : localRanges)
|
||||
{
|
||||
// WeightedRange supports only unwrapped ranges as it relies
|
||||
// on right - left == num tokens equality
|
||||
for (Range<Token> u: r.unwrap())
|
||||
weightedRanges.add(new Splitter.WeightedRange(1.0, u));
|
||||
}
|
||||
weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
|
||||
}
|
||||
else
|
||||
{
|
||||
weightedRanges = fullWeightedRange();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
weightedRanges = new ArrayList<>(localRanges.size());
|
||||
for (Range<Token> r : localRanges)
|
||||
{
|
||||
// WeightedRange supports only unwrapped ranges as it relies
|
||||
// on right - left == num tokens equality
|
||||
for (Range<Token> u: r.unwrap())
|
||||
weightedRanges.add(new Splitter.WeightedRange(1.0, u));
|
||||
}
|
||||
weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
|
||||
// Local tables need to cover the full token range and don't care about ring changes.
|
||||
// We also end up here if the table's partitioner is not the database's, which can happen in tests.
|
||||
weightedRanges = fullWeightedRange();
|
||||
ringVersion = -1;
|
||||
}
|
||||
|
||||
List<Token> boundaries = getPartitioner().splitter().get().splitOwnedRanges(shardCount, weightedRanges, false);
|
||||
shardBoundaries = new ShardBoundaries(boundaries.subList(0, boundaries.size() - 1),
|
||||
versionedLocalRanges.ringVersion);
|
||||
ringVersion);
|
||||
cachedShardBoundaries = shardBoundaries;
|
||||
logger.debug("Memtable shard boundaries for {}.{}: {}", keyspace.getName(), getTableName(), boundaries);
|
||||
}
|
||||
return shardBoundaries;
|
||||
}
|
||||
|
||||
private ImmutableList<Splitter.WeightedRange> fullWeightedRange()
|
||||
{
|
||||
return ImmutableList.of(new Splitter.WeightedRange(1.0, new Range<>(getPartitioner().getMinimumToken(), getPartitioner().getMaximumToken())));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param sstables
|
||||
* @return sstables whose key range overlaps with that of the given sstables, not including itself.
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ public abstract class DecoratedKey implements PartitionPosition, FilterKey
|
|||
}
|
||||
|
||||
public abstract ByteBuffer getKey();
|
||||
public abstract int getKeyLength();
|
||||
|
||||
public void filterHash(long[] dest)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import org.apache.cassandra.db.marshal.ByteBufferAccessor;
|
|||
import org.apache.cassandra.db.marshal.ValueAccessor;
|
||||
import org.apache.cassandra.utils.ObjectSizes;
|
||||
import org.apache.cassandra.utils.concurrent.OpOrder;
|
||||
import org.apache.cassandra.utils.memory.HeapCloner;
|
||||
import org.apache.cassandra.utils.memory.MemoryUtil;
|
||||
import org.apache.cassandra.utils.memory.NativeAllocator;
|
||||
|
||||
|
|
@ -36,11 +37,6 @@ public class NativeClustering implements Clustering<ByteBuffer>
|
|||
|
||||
private NativeClustering() { peer = 0; }
|
||||
|
||||
public ClusteringPrefix<ByteBuffer> minimize()
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
public NativeClustering(NativeAllocator allocator, OpOrder.Group writeOp, Clustering<?> clustering)
|
||||
{
|
||||
int count = clustering.size();
|
||||
|
|
@ -157,4 +153,20 @@ public class NativeClustering implements Clustering<ByteBuffer>
|
|||
{
|
||||
return ClusteringPrefix.equals(this, o);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ClusteringPrefix<ByteBuffer> retainable()
|
||||
{
|
||||
assert kind() == Kind.CLUSTERING; // tombstones are never stored natively
|
||||
|
||||
// always extract
|
||||
ByteBuffer[] values = new ByteBuffer[size()];
|
||||
for (int i = 0; i < values.length; ++i)
|
||||
{
|
||||
ByteBuffer value = get(i);
|
||||
values[i] = value != null ? HeapCloner.instance.clone(value) : null;
|
||||
}
|
||||
|
||||
return accessor().factory().clustering(values);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -72,6 +72,12 @@ public class NativeDecoratedKey extends DecoratedKey
|
|||
return MemoryUtil.getByteBuffer(address(), length(), ByteOrder.BIG_ENDIAN);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getKeyLength()
|
||||
{
|
||||
return MemoryUtil.getInt(peer);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ByteSource keyComparableBytes(Version version)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -67,6 +67,9 @@ public abstract class AbstractAllocatorMemtable extends AbstractMemtableWithComm
|
|||
// memtable was created with the new or old comparator.
|
||||
@Unmetered
|
||||
protected final ClusteringComparator initialComparator;
|
||||
// As above, used to determine if the memtable needs to be flushed on schema change.
|
||||
@Unmetered
|
||||
public final Factory initialFactory;
|
||||
|
||||
private final long creationNano = Clock.Global.nanoTime();
|
||||
|
||||
|
|
@ -104,6 +107,7 @@ public abstract class AbstractAllocatorMemtable extends AbstractMemtableWithComm
|
|||
super(metadataRef, commitLogLowerBound);
|
||||
this.allocator = MEMORY_POOL.newAllocator(metadataRef.toString());
|
||||
this.initialComparator = metadata.get().comparator;
|
||||
this.initialFactory = metadata().params.memtable.factory();
|
||||
this.owner = owner;
|
||||
scheduleFlush();
|
||||
}
|
||||
|
|
@ -120,7 +124,7 @@ public abstract class AbstractAllocatorMemtable extends AbstractMemtableWithComm
|
|||
{
|
||||
case SCHEMA_CHANGE:
|
||||
return initialComparator != metadata().comparator // If the CF comparator has changed, because our partitions reference the old one
|
||||
|| metadata().params.memtable.factory() != factory(); // If a different type of memtable is requested
|
||||
|| !initialFactory.equals(metadata().params.memtable.factory()); // If a different type of memtable is requested
|
||||
case OWNED_RANGES_CHANGE:
|
||||
return false; // by default we don't use the local ranges, thus this has no effect
|
||||
default:
|
||||
|
|
@ -145,8 +149,6 @@ public abstract class AbstractAllocatorMemtable extends AbstractMemtableWithComm
|
|||
throw new AssertionError("performSnapshot must be implemented if shouldSwitch(SNAPSHOT) can return false.");
|
||||
}
|
||||
|
||||
protected abstract Factory factory();
|
||||
|
||||
public void switchOut(OpOrder.Barrier writeBarrier, AtomicReference<CommitLogPosition> commitLogUpperBound)
|
||||
{
|
||||
super.switchOut(writeBarrier, commitLogUpperBound);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,103 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.db.memtable;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.cassandra.db.commitlog.CommitLogPosition;
|
||||
import org.apache.cassandra.schema.TableMetadataRef;
|
||||
import org.apache.cassandra.utils.FBUtilities;
|
||||
import org.apache.cassandra.utils.MBeanWrapper;
|
||||
import org.github.jamm.Unmetered;
|
||||
|
||||
public abstract class AbstractShardedMemtable extends AbstractAllocatorMemtable
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(AbstractShardedMemtable.class);
|
||||
|
||||
public static final String SHARDS_OPTION = "shards";
|
||||
@VisibleForTesting
|
||||
public static final String DEFAULT_SHARD_COUNT_PROPERTY = "cassandra.memtable.shard.count";
|
||||
|
||||
public static final String SHARDED_MEMTABLE_CONFIG_OBJECT_NAME = "org.apache.cassandra.db:type=ShardedMemtableConfig";
|
||||
static
|
||||
{
|
||||
MBeanWrapper.instance.registerMBean(new ShardedMemtableConfig(), SHARDED_MEMTABLE_CONFIG_OBJECT_NAME, MBeanWrapper.OnException.LOG);
|
||||
}
|
||||
|
||||
// default shard count, used when a specific number of shards is not specified in the options
|
||||
private static volatile int defaultShardCount = Integer.getInteger(DEFAULT_SHARD_COUNT_PROPERTY, FBUtilities.getAvailableProcessors());
|
||||
|
||||
// The boundaries for the keyspace as they were calculated when the memtable is created.
|
||||
// The boundaries will be NONE for system keyspaces or if StorageService is not yet initialized.
|
||||
// The fact this is fixed for the duration of the memtable lifetime, guarantees we'll always pick the same shard
|
||||
// for a given key, even if we race with the StorageService initialization or with topology changes.
|
||||
@Unmetered
|
||||
protected final ShardBoundaries boundaries;
|
||||
|
||||
AbstractShardedMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound,
|
||||
TableMetadataRef metadataRef,
|
||||
Owner owner,
|
||||
Integer shardCountOption)
|
||||
{
|
||||
super(commitLogLowerBound, metadataRef, owner);
|
||||
int shardCount = shardCountOption != null ? shardCountOption : defaultShardCount;
|
||||
this.boundaries = owner.localRangeSplits(shardCount);
|
||||
}
|
||||
|
||||
private static class ShardedMemtableConfig implements ShardedMemtableConfigMXBean
|
||||
{
|
||||
@Override
|
||||
public void setDefaultShardCount(String shardCount)
|
||||
{
|
||||
if ("auto".equalsIgnoreCase(shardCount))
|
||||
{
|
||||
defaultShardCount = FBUtilities.getAvailableProcessors();
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
defaultShardCount = Integer.parseInt(shardCount);
|
||||
}
|
||||
catch (NumberFormatException ex)
|
||||
{
|
||||
logger.warn("Unable to parse {} as valid value for shard count", shardCount);
|
||||
return;
|
||||
}
|
||||
}
|
||||
logger.info("Requested setting shard count to {}; set to: {}", shardCount, defaultShardCount);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDefaultShardCount()
|
||||
{
|
||||
return Integer.toString(defaultShardCount);
|
||||
}
|
||||
}
|
||||
|
||||
public static int getDefaultShardCount()
|
||||
{
|
||||
return defaultShardCount;
|
||||
}
|
||||
}
|
||||
|
|
@ -45,13 +45,32 @@ memtable:
|
|||
class_name: SkipListMemtable
|
||||
sharded:
|
||||
class_name: ShardedSkipListMemtable
|
||||
trie:
|
||||
class_name: TrieMemtable
|
||||
default:
|
||||
inherits: sharded
|
||||
inherits: trie
|
||||
```
|
||||
|
||||
Note that the database will only validate the memtable class and its parameters when a configuration needs to be
|
||||
instantiated for a table.
|
||||
|
||||
## Implementations provided
|
||||
|
||||
Cassandra currently comes with three memtable implementations:
|
||||
|
||||
- `SkipListMemtable` is the default and matches the memtable format of Cassandra versions up to 4.1. It organizes
|
||||
partitions into a single concurrent skip list.
|
||||
- `ShardedSkipListMemtable` splits the partition skip-list into several independent skip-lists each covering a roughly
|
||||
equal part of the token space served by this node. This reduces congestion of the skip-list from concurrent writes and
|
||||
can lead to improved write throughput. Its configuration takes two parameters:
|
||||
- `shards`: the number of shards to split into, defaulting to the number of CPU cores on the machine.
|
||||
- `serialize_writes`: if false (default), each shard may serve multiple writes in parallel; if true, writes to each
|
||||
shard are synchronized.
|
||||
- `TrieMemtable` is a novel solution that organizes partitions into an in-memory trie which places the partition
|
||||
indexing structure in a buffer, off-heap if desired, which significantly improves garbage collection efficiency. It
|
||||
also improves the memtable's space efficiency and lookup performance. Its configuration can take a single parameter
|
||||
`shards` as above.
|
||||
|
||||
## Memtable selection
|
||||
|
||||
Once a configuration has been defined, it can be used by specifying it in the `memtable` parameter of a `CREATE TABLE`
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import java.util.List;
|
|||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.db.Keyspace;
|
||||
import org.apache.cassandra.db.PartitionPosition;
|
||||
import org.apache.cassandra.dht.Token;
|
||||
|
|
@ -83,7 +82,7 @@ public class ShardBoundaries
|
|||
if (boundaries.length == 0)
|
||||
return 0;
|
||||
|
||||
assert (key.getPartitioner() == DatabaseDescriptor.getPartitioner());
|
||||
assert (key.getPartitioner() == boundaries[0].getPartitioner());
|
||||
return getShardForToken(key.getToken());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,28 +16,18 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.db;
|
||||
package org.apache.cassandra.db.memtable;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class ClusteringBoundTest
|
||||
public interface ShardedMemtableConfigMXBean
|
||||
{
|
||||
@Test
|
||||
public void arrayTopAndBottom()
|
||||
{
|
||||
Assert.assertTrue(ArrayClusteringBound.BOTTOM.isBottom());
|
||||
Assert.assertFalse(ArrayClusteringBound.BOTTOM.isTop());
|
||||
Assert.assertTrue(ArrayClusteringBound.TOP.isTop());
|
||||
Assert.assertFalse(ArrayClusteringBound.TOP.isBottom());
|
||||
}
|
||||
/**
|
||||
* Adjust the shard count for sharded memtables that do not specify it explicitly in the memtable options.
|
||||
* Changes will apply on the next memtable flush.
|
||||
*/
|
||||
public void setDefaultShardCount(String numShards);
|
||||
|
||||
@Test
|
||||
public void bufferTopAndBottom()
|
||||
{
|
||||
Assert.assertTrue(BufferClusteringBound.BOTTOM.isBottom());
|
||||
Assert.assertFalse(BufferClusteringBound.BOTTOM.isTop());
|
||||
Assert.assertTrue(BufferClusteringBound.TOP.isTop());
|
||||
Assert.assertFalse(BufferClusteringBound.TOP.isBottom());
|
||||
}
|
||||
/**
|
||||
* Returns the shard count for sharded memtables that do not specify it explicitly in the memtable options.
|
||||
*/
|
||||
public String getDefaultShardCount();
|
||||
}
|
||||
|
|
@ -41,6 +41,7 @@ import org.apache.cassandra.db.filter.ClusteringIndexFilter;
|
|||
import org.apache.cassandra.db.filter.ColumnFilter;
|
||||
import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
|
||||
import org.apache.cassandra.db.partitions.AtomicBTreePartition;
|
||||
import org.apache.cassandra.db.partitions.BTreePartitionUpdater;
|
||||
import org.apache.cassandra.db.partitions.Partition;
|
||||
import org.apache.cassandra.db.partitions.PartitionUpdate;
|
||||
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
|
||||
|
|
@ -54,7 +55,6 @@ import org.apache.cassandra.index.transactions.UpdateTransaction;
|
|||
import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
|
||||
import org.apache.cassandra.schema.TableMetadata;
|
||||
import org.apache.cassandra.schema.TableMetadataRef;
|
||||
import org.apache.cassandra.utils.FBUtilities;
|
||||
import org.apache.cassandra.utils.concurrent.OpOrder;
|
||||
import org.apache.cassandra.utils.memory.Cloner;
|
||||
import org.apache.cassandra.utils.memory.MemtableAllocator;
|
||||
|
|
@ -72,46 +72,26 @@ import org.github.jamm.Unmetered;
|
|||
*
|
||||
* Also see Memtable_API.md.
|
||||
*/
|
||||
public class ShardedSkipListMemtable extends AbstractAllocatorMemtable
|
||||
public class ShardedSkipListMemtable extends AbstractShardedMemtable
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(ShardedSkipListMemtable.class);
|
||||
|
||||
public static final String SHARDS_OPTION = "shards";
|
||||
public static final String LOCKING_OPTION = "serialize_writes";
|
||||
|
||||
// The boundaries for the keyspace as they were calculated when the memtable is created.
|
||||
// The boundaries will be NONE for system keyspaces or if StorageService is not yet initialized.
|
||||
// The fact this is fixed for the duration of the memtable lifetime, guarantees we'll always pick the same shard
|
||||
// for a given key, even if we race with the StorageService initialization or with topology changes.
|
||||
@Unmetered
|
||||
final ShardBoundaries boundaries;
|
||||
|
||||
/**
|
||||
* Core-specific memtable regions. All writes must go through the specific core. The data structures used
|
||||
* are concurrent-read safe, thus reads can be carried out from any thread.
|
||||
* Sharded memtable sections. Each is responsible for a contiguous range of the token space (between boundaries[i]
|
||||
* and boundaries[i+1]) and is written to by one thread at a time, while reads are carried out concurrently
|
||||
* (including with any write).
|
||||
*/
|
||||
final MemtableShard[] shards;
|
||||
|
||||
@VisibleForTesting
|
||||
public static final String SHARD_COUNT_PROPERTY = "cassandra.memtable.shard.count";
|
||||
|
||||
// default shard count, used when a specific number of shards is not specified in the parameters
|
||||
private static final int SHARD_COUNT = Integer.getInteger(SHARD_COUNT_PROPERTY, FBUtilities.getAvailableProcessors());
|
||||
|
||||
private final Factory factory;
|
||||
|
||||
// only to be used by init(), to setup the very first memtable for the cfs
|
||||
ShardedSkipListMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound,
|
||||
TableMetadataRef metadataRef,
|
||||
Owner owner,
|
||||
Integer shardCountOption,
|
||||
Factory factory)
|
||||
Integer shardCountOption)
|
||||
{
|
||||
super(commitLogLowerBound, metadataRef, owner);
|
||||
int shardCount = shardCountOption != null ? shardCountOption : SHARD_COUNT;
|
||||
this.boundaries = owner.localRangeSplits(shardCount);
|
||||
super(commitLogLowerBound, metadataRef, owner, shardCountOption);
|
||||
this.shards = generatePartitionShards(boundaries.shardCount(), allocator, metadataRef);
|
||||
this.factory = factory;
|
||||
}
|
||||
|
||||
private static MemtableShard[] generatePartitionShards(int splits,
|
||||
|
|
@ -128,17 +108,11 @@ public class ShardedSkipListMemtable extends AbstractAllocatorMemtable
|
|||
public boolean isClean()
|
||||
{
|
||||
for (MemtableShard shard : shards)
|
||||
if (!shard.isEmpty())
|
||||
if (!shard.isClean())
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Memtable.Factory factory()
|
||||
{
|
||||
return factory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate
|
||||
* OpOrdering.
|
||||
|
|
@ -389,14 +363,14 @@ public class ShardedSkipListMemtable extends AbstractAllocatorMemtable
|
|||
}
|
||||
}
|
||||
|
||||
long[] pair = previous.addAllWithSizeDelta(update, cloner, opGroup, indexer);
|
||||
BTreePartitionUpdater updater = previous.addAll(update, cloner, opGroup, indexer);
|
||||
updateMin(minTimestamp, update.stats().minTimestamp);
|
||||
updateMin(minLocalDeletionTime, update.stats().minLocalDeletionTime);
|
||||
liveDataSize.addAndGet(initialSize + pair[0]);
|
||||
liveDataSize.addAndGet(initialSize + updater.dataSize);
|
||||
columnsCollector.update(update.columns());
|
||||
statsCollector.update(update.stats());
|
||||
currentOperations.addAndGet(update.operationCount());
|
||||
return pair[1];
|
||||
return updater.colUpdateTimeDelta;
|
||||
}
|
||||
|
||||
private Map<PartitionPosition, AtomicBTreePartition> getPartitionsSubMap(PartitionPosition left,
|
||||
|
|
@ -425,7 +399,7 @@ public class ShardedSkipListMemtable extends AbstractAllocatorMemtable
|
|||
}
|
||||
}
|
||||
|
||||
public boolean isEmpty()
|
||||
public boolean isClean()
|
||||
{
|
||||
return partitions.isEmpty();
|
||||
}
|
||||
|
|
@ -493,9 +467,9 @@ public class ShardedSkipListMemtable extends AbstractAllocatorMemtable
|
|||
|
||||
static class Locking extends ShardedSkipListMemtable
|
||||
{
|
||||
Locking(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner, Integer shardCountOption, Factory factory)
|
||||
Locking(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner, Integer shardCountOption)
|
||||
{
|
||||
super(commitLogLowerBound, metadataRef, owner, shardCountOption, factory);
|
||||
super(commitLogLowerBound, metadataRef, owner, shardCountOption);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -540,8 +514,8 @@ public class ShardedSkipListMemtable extends AbstractAllocatorMemtable
|
|||
Owner owner)
|
||||
{
|
||||
return isLocking
|
||||
? new Locking(commitLogLowerBound, metadataRef, owner, shardCount, this)
|
||||
: new ShardedSkipListMemtable(commitLogLowerBound, metadataRef, owner, shardCount, this);
|
||||
? new Locking(commitLogLowerBound, metadataRef, owner, shardCount)
|
||||
: new ShardedSkipListMemtable(commitLogLowerBound, metadataRef, owner, shardCount);
|
||||
}
|
||||
|
||||
public boolean equals(Object o)
|
||||
|
|
|
|||
|
|
@ -37,9 +37,10 @@ import org.apache.cassandra.db.Slices;
|
|||
import org.apache.cassandra.db.commitlog.CommitLogPosition;
|
||||
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
|
||||
import org.apache.cassandra.db.filter.ColumnFilter;
|
||||
import org.apache.cassandra.db.partitions.AbstractBTreePartition;
|
||||
import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
|
||||
import org.apache.cassandra.db.partitions.AtomicBTreePartition;
|
||||
import org.apache.cassandra.db.partitions.BTreePartitionData;
|
||||
import org.apache.cassandra.db.partitions.BTreePartitionUpdater;
|
||||
import org.apache.cassandra.db.partitions.Partition;
|
||||
import org.apache.cassandra.db.partitions.PartitionUpdate;
|
||||
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
|
||||
|
|
@ -90,12 +91,6 @@ public class SkipListMemtable extends AbstractAllocatorMemtable
|
|||
super(commitLogLowerBound, metadataRef, owner);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Factory factory()
|
||||
{
|
||||
return FACTORY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isClean()
|
||||
{
|
||||
|
|
@ -132,14 +127,14 @@ public class SkipListMemtable extends AbstractAllocatorMemtable
|
|||
}
|
||||
}
|
||||
|
||||
long[] pair = previous.addAllWithSizeDelta(update, cloner, opGroup, indexer);
|
||||
BTreePartitionUpdater updater = previous.addAll(update, cloner, opGroup, indexer);
|
||||
updateMin(minTimestamp, update.stats().minTimestamp);
|
||||
updateMin(minLocalDeletionTime, update.stats().minLocalDeletionTime);
|
||||
liveDataSize.addAndGet(initialSize + pair[0]);
|
||||
liveDataSize.addAndGet(initialSize + updater.dataSize);
|
||||
columnsCollector.update(update.columns());
|
||||
statsCollector.update(update.stats());
|
||||
currentOperations.addAndGet(update.operationCount());
|
||||
return pair[1];
|
||||
return updater.colUpdateTimeDelta;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -235,7 +230,7 @@ public class SkipListMemtable extends AbstractAllocatorMemtable
|
|||
rowOverhead = (int) ((avgSize - Math.floor(avgSize)) < 0.05 ? Math.floor(avgSize) : Math.ceil(avgSize));
|
||||
rowOverhead -= new LongToken(0).getHeapSize();
|
||||
rowOverhead += AtomicBTreePartition.EMPTY_SIZE;
|
||||
rowOverhead += AbstractBTreePartition.HOLDER_UNSHARED_HEAP_SIZE;
|
||||
rowOverhead += BTreePartitionData.UNSHARED_HEAP_SIZE;
|
||||
if (!(allocator instanceof NativeAllocator))
|
||||
rowOverhead -= testBufferSize; // measureDeepOmitShared includes the given number of bytes even for
|
||||
// off-heap buffers, but not for direct memory.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,722 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.db.memtable;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.NavigableSet;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.Iterators;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.db.BufferDecoratedKey;
|
||||
import org.apache.cassandra.db.Clustering;
|
||||
import org.apache.cassandra.db.ColumnFamilyStore;
|
||||
import org.apache.cassandra.db.DataRange;
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.db.DeletionInfo;
|
||||
import org.apache.cassandra.db.PartitionPosition;
|
||||
import org.apache.cassandra.db.RegularAndStaticColumns;
|
||||
import org.apache.cassandra.db.Slices;
|
||||
import org.apache.cassandra.db.commitlog.CommitLogPosition;
|
||||
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
|
||||
import org.apache.cassandra.db.filter.ColumnFilter;
|
||||
import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator;
|
||||
import org.apache.cassandra.db.partitions.BTreePartitionData;
|
||||
import org.apache.cassandra.db.partitions.BTreePartitionUpdater;
|
||||
import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
|
||||
import org.apache.cassandra.db.partitions.Partition;
|
||||
import org.apache.cassandra.db.partitions.PartitionUpdate;
|
||||
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
|
||||
import org.apache.cassandra.db.rows.EncodingStats;
|
||||
import org.apache.cassandra.db.rows.Row;
|
||||
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
|
||||
import org.apache.cassandra.db.tries.MemtableTrie;
|
||||
import org.apache.cassandra.db.tries.Trie;
|
||||
import org.apache.cassandra.dht.AbstractBounds;
|
||||
import org.apache.cassandra.dht.Bounds;
|
||||
import org.apache.cassandra.dht.IncludingExcludingBounds;
|
||||
import org.apache.cassandra.dht.Range;
|
||||
import org.apache.cassandra.index.transactions.UpdateTransaction;
|
||||
import org.apache.cassandra.io.compress.BufferType;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
|
||||
import org.apache.cassandra.metrics.TableMetrics;
|
||||
import org.apache.cassandra.metrics.TrieMemtableMetricsView;
|
||||
import org.apache.cassandra.schema.TableMetadata;
|
||||
import org.apache.cassandra.schema.TableMetadataRef;
|
||||
import org.apache.cassandra.utils.Clock;
|
||||
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
|
||||
import org.apache.cassandra.utils.bytecomparable.ByteSource;
|
||||
import org.apache.cassandra.utils.concurrent.OpOrder;
|
||||
import org.apache.cassandra.utils.memory.EnsureOnHeap;
|
||||
import org.apache.cassandra.utils.memory.MemtableAllocator;
|
||||
import org.github.jamm.Unmetered;
|
||||
|
||||
/**
|
||||
* Trie memtable implementation. Improves memory usage, garbage collection efficiency and lookup performance.
|
||||
* The implementation is described in detail in the paper:
|
||||
* https://www.vldb.org/pvldb/vol15/p3359-lambov.pdf
|
||||
*
|
||||
* The configuration takes a single parameter:
|
||||
* - shards: the number of shards to split into, defaulting to the number of CPU cores.
|
||||
*
|
||||
* Also see Memtable_API.md.
|
||||
*/
|
||||
public class TrieMemtable extends AbstractShardedMemtable
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(TrieMemtable.class);
|
||||
|
||||
/** Buffer type to use for memtable tries (on- vs off-heap) */
|
||||
public static final BufferType BUFFER_TYPE;
|
||||
|
||||
static
|
||||
{
|
||||
switch (DatabaseDescriptor.getMemtableAllocationType())
|
||||
{
|
||||
case unslabbed_heap_buffers:
|
||||
case heap_buffers:
|
||||
BUFFER_TYPE = BufferType.ON_HEAP;
|
||||
break;
|
||||
case offheap_buffers:
|
||||
case offheap_objects:
|
||||
BUFFER_TYPE = BufferType.OFF_HEAP;
|
||||
break;
|
||||
default:
|
||||
throw new AssertionError();
|
||||
}
|
||||
}
|
||||
|
||||
/** If keys is below this length, we will use a recursive procedure for inserting data in the memtable trie. */
|
||||
@VisibleForTesting
|
||||
public static final int MAX_RECURSIVE_KEY_LENGTH = 128;
|
||||
|
||||
/** The byte-ordering conversion version to use for memtables. */
|
||||
public static final ByteComparable.Version BYTE_COMPARABLE_VERSION = ByteComparable.Version.OSS42;
|
||||
|
||||
// Set to true when the memtable requests a switch (e.g. for trie size limit being reached) to ensure only one
|
||||
// thread calls cfs.switchMemtableIfCurrent.
|
||||
private final AtomicBoolean switchRequested = new AtomicBoolean(false);
|
||||
|
||||
/**
|
||||
* Sharded memtable sections. Each is responsible for a contiguous range of the token space (between boundaries[i]
|
||||
* and boundaries[i+1]) and is written to by one thread at a time, while reads are carried out concurrently
|
||||
* (including with any write).
|
||||
*/
|
||||
private final MemtableShard[] shards;
|
||||
|
||||
/**
|
||||
* A merged view of the memtable map. Used for partition range queries and flush.
|
||||
* For efficiency we serve single partition requests off the shard which offers more direct MemtableTrie methods.
|
||||
*/
|
||||
private final Trie<BTreePartitionData> mergedTrie;
|
||||
|
||||
@Unmetered
|
||||
private final TrieMemtableMetricsView metrics;
|
||||
|
||||
TrieMemtable(AtomicReference<CommitLogPosition> commitLogLowerBound, TableMetadataRef metadataRef, Owner owner, Integer shardCountOption)
|
||||
{
|
||||
super(commitLogLowerBound, metadataRef, owner, shardCountOption);
|
||||
this.metrics = new TrieMemtableMetricsView(metadataRef.keyspace, metadataRef.name);
|
||||
this.shards = generatePartitionShards(boundaries.shardCount(), allocator, metadataRef, metrics);
|
||||
this.mergedTrie = makeMergedTrie(shards);
|
||||
}
|
||||
|
||||
private static MemtableShard[] generatePartitionShards(int splits,
|
||||
MemtableAllocator allocator,
|
||||
TableMetadataRef metadata,
|
||||
TrieMemtableMetricsView metrics)
|
||||
{
|
||||
MemtableShard[] partitionMapContainer = new MemtableShard[splits];
|
||||
for (int i = 0; i < splits; i++)
|
||||
partitionMapContainer[i] = new MemtableShard(metadata, allocator, metrics);
|
||||
|
||||
return partitionMapContainer;
|
||||
}
|
||||
|
||||
private static Trie<BTreePartitionData> makeMergedTrie(MemtableShard[] shards)
|
||||
{
|
||||
List<Trie<BTreePartitionData>> tries = new ArrayList<>(shards.length);
|
||||
for (MemtableShard shard : shards)
|
||||
tries.add(shard.data);
|
||||
return Trie.mergeDistinct(tries);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isClean()
|
||||
{
|
||||
for (MemtableShard shard : shards)
|
||||
if (!shard.isClean())
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void discard()
|
||||
{
|
||||
super.discard();
|
||||
// metrics here are not thread safe, but I think we can live with that
|
||||
metrics.lastFlushShardDataSizes.reset();
|
||||
for (MemtableShard shard : shards)
|
||||
{
|
||||
metrics.lastFlushShardDataSizes.update(shard.liveDataSize());
|
||||
}
|
||||
// the buffer release is a longer-running process, do it in a separate loop to not make the metrics update wait
|
||||
for (MemtableShard shard : shards)
|
||||
{
|
||||
shard.data.discardBuffers();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate
|
||||
* OpOrdering.
|
||||
*
|
||||
* commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null
|
||||
*/
|
||||
@Override
|
||||
public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup)
|
||||
{
|
||||
try
|
||||
{
|
||||
DecoratedKey key = update.partitionKey();
|
||||
MemtableShard shard = shards[boundaries.getShardForKey(key)];
|
||||
long colUpdateTimeDelta = shard.put(key, update, indexer, opGroup);
|
||||
|
||||
if (shard.data.reachedAllocatedSizeThreshold() && !switchRequested.getAndSet(true))
|
||||
{
|
||||
logger.info("Scheduling flush due to trie size limit reached.");
|
||||
owner.signalFlushRequired(this, ColumnFamilyStore.FlushReason.MEMTABLE_LIMIT);
|
||||
}
|
||||
|
||||
return colUpdateTimeDelta;
|
||||
}
|
||||
catch (MemtableTrie.SpaceExhaustedException e)
|
||||
{
|
||||
// This should never happen as {@link MemtableTrie#reachedAllocatedSizeThreshold} should become
|
||||
// true and trigger a memtable switch long before this limit is reached.
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Technically we should scatter gather on all the core threads because the size in following calls are not
|
||||
* using volatile variables, but for metrics purpose this should be good enough.
|
||||
*/
|
||||
@Override
|
||||
public long getLiveDataSize()
|
||||
{
|
||||
long total = 0L;
|
||||
for (MemtableShard shard : shards)
|
||||
total += shard.liveDataSize();
|
||||
return total;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long operationCount()
|
||||
{
|
||||
long total = 0L;
|
||||
for (MemtableShard shard : shards)
|
||||
total += shard.currentOperations();
|
||||
return total;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long partitionCount()
|
||||
{
|
||||
int total = 0;
|
||||
for (MemtableShard shard : shards)
|
||||
total += shard.size();
|
||||
return total;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMinTimestamp()
|
||||
{
|
||||
long min = Long.MAX_VALUE;
|
||||
for (MemtableShard shard : shards)
|
||||
min = Long.min(min, shard.minTimestamp());
|
||||
return min;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getMinLocalDeletionTime()
|
||||
{
|
||||
int min = Integer.MAX_VALUE;
|
||||
for (MemtableShard shard : shards)
|
||||
min = Integer.min(min, shard.minLocalDeletionTime());
|
||||
return min;
|
||||
}
|
||||
|
||||
@Override
|
||||
RegularAndStaticColumns columns()
|
||||
{
|
||||
for (MemtableShard shard : shards)
|
||||
columnsCollector.update(shard.columnsCollector);
|
||||
return columnsCollector.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
EncodingStats encodingStats()
|
||||
{
|
||||
for (MemtableShard shard : shards)
|
||||
statsCollector.update(shard.statsCollector.get());
|
||||
return statsCollector.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public MemtableUnfilteredPartitionIterator partitionIterator(final ColumnFilter columnFilter,
|
||||
final DataRange dataRange,
|
||||
SSTableReadsListener readsListener)
|
||||
{
|
||||
AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
|
||||
|
||||
PartitionPosition left = keyRange.left;
|
||||
PartitionPosition right = keyRange.right;
|
||||
if (left.isMinimum())
|
||||
left = null;
|
||||
if (right.isMinimum())
|
||||
right = null;
|
||||
|
||||
boolean isBound = keyRange instanceof Bounds;
|
||||
boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds;
|
||||
boolean includeStop = isBound || keyRange instanceof Range;
|
||||
|
||||
Trie<BTreePartitionData> subMap = mergedTrie.subtrie(left, includeStart, right, includeStop);
|
||||
|
||||
return new MemtableUnfilteredPartitionIterator(metadata(),
|
||||
allocator.ensureOnHeap(),
|
||||
subMap,
|
||||
columnFilter,
|
||||
dataRange);
|
||||
// readsListener is ignored as it only accepts sstable signals
|
||||
}
|
||||
|
||||
private Partition getPartition(DecoratedKey key)
|
||||
{
|
||||
int shardIndex = boundaries.getShardForKey(key);
|
||||
BTreePartitionData data = shards[shardIndex].data.get(key);
|
||||
if (data != null)
|
||||
return createPartition(metadata(), allocator.ensureOnHeap(), key, data);
|
||||
else
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener)
|
||||
{
|
||||
Partition p = getPartition(key);
|
||||
if (p == null)
|
||||
return null;
|
||||
else
|
||||
return p.unfilteredIterator(selectedColumns, slices, reversed);
|
||||
}
|
||||
|
||||
@Override
|
||||
public UnfilteredRowIterator rowIterator(DecoratedKey key)
|
||||
{
|
||||
Partition p = getPartition(key);
|
||||
return p != null ? p.unfilteredIterator() : null;
|
||||
}
|
||||
|
||||
private static MemtablePartition createPartition(TableMetadata metadata, EnsureOnHeap ensureOnHeap, DecoratedKey key, BTreePartitionData data)
|
||||
{
|
||||
return new MemtablePartition(metadata, ensureOnHeap, key, data);
|
||||
}
|
||||
|
||||
private static MemtablePartition getPartitionFromTrieEntry(TableMetadata metadata, EnsureOnHeap ensureOnHeap, Map.Entry<ByteComparable, BTreePartitionData> en)
|
||||
{
|
||||
DecoratedKey key = BufferDecoratedKey.fromByteComparable(en.getKey(),
|
||||
BYTE_COMPARABLE_VERSION,
|
||||
metadata.partitioner);
|
||||
return createPartition(metadata, ensureOnHeap, key, en.getValue());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public FlushablePartitionSet<MemtablePartition> getFlushSet(PartitionPosition from, PartitionPosition to)
|
||||
{
|
||||
Trie<BTreePartitionData> toFlush = mergedTrie.subtrie(from, true, to, false);
|
||||
long keySize = 0;
|
||||
int keyCount = 0;
|
||||
|
||||
for (Iterator<Map.Entry<ByteComparable, BTreePartitionData>> it = toFlush.entryIterator(); it.hasNext(); )
|
||||
{
|
||||
Map.Entry<ByteComparable, BTreePartitionData> en = it.next();
|
||||
byte[] keyBytes = DecoratedKey.keyFromByteSource(ByteSource.peekable(en.getKey().asComparableBytes(BYTE_COMPARABLE_VERSION)),
|
||||
BYTE_COMPARABLE_VERSION,
|
||||
metadata().partitioner);
|
||||
keySize += keyBytes.length;
|
||||
keyCount++;
|
||||
}
|
||||
long partitionKeySize = keySize;
|
||||
int partitionCount = keyCount;
|
||||
|
||||
return new AbstractFlushablePartitionSet<MemtablePartition>()
|
||||
{
|
||||
public Memtable memtable()
|
||||
{
|
||||
return TrieMemtable.this;
|
||||
}
|
||||
|
||||
public PartitionPosition from()
|
||||
{
|
||||
return from;
|
||||
}
|
||||
|
||||
public PartitionPosition to()
|
||||
{
|
||||
return to;
|
||||
}
|
||||
|
||||
public long partitionCount()
|
||||
{
|
||||
return partitionCount;
|
||||
}
|
||||
|
||||
public Iterator<MemtablePartition> iterator()
|
||||
{
|
||||
return Iterators.transform(toFlush.entryIterator(),
|
||||
// During flushing we are certain the memtable will remain at least until
|
||||
// the flush completes. No copying to heap is necessary.
|
||||
entry -> getPartitionFromTrieEntry(metadata(), EnsureOnHeap.NOOP, entry));
|
||||
}
|
||||
|
||||
public long partitionKeysSize()
|
||||
{
|
||||
return partitionKeySize;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
static class MemtableShard
|
||||
{
|
||||
// The following fields are volatile as we have to make sure that when we
|
||||
// collect results from all sub-ranges, the thread accessing the value
|
||||
// is guaranteed to see the changes to the values.
|
||||
|
||||
// The smallest timestamp for all partitions stored in this shard
|
||||
private volatile long minTimestamp = Long.MAX_VALUE;
|
||||
|
||||
private volatile int minLocalDeletionTime = Integer.MAX_VALUE;
|
||||
|
||||
private volatile long liveDataSize = 0;
|
||||
|
||||
private volatile long currentOperations = 0;
|
||||
|
||||
@Unmetered
|
||||
private final ReentrantLock writeLock = new ReentrantLock();
|
||||
|
||||
// Content map for the given shard. This is implemented as a memtable trie which uses the prefix-free
|
||||
// byte-comparable ByteSource representations of the keys to address the partitions.
|
||||
//
|
||||
// This map is used in a single-producer, multi-consumer fashion: only one thread will insert items but
|
||||
// several threads may read from it and iterate over it. Iterators (especially partition range iterators)
|
||||
// may operate for a long period of time and thus iterators should not throw ConcurrentModificationExceptions
|
||||
// if the underlying map is modified during iteration, they should provide a weakly consistent view of the map
|
||||
// instead.
|
||||
//
|
||||
// Also, this data is backed by memtable memory, when accessing it callers must specify if it can be accessed
|
||||
// unsafely, meaning that the memtable will not be discarded as long as the data is used, or whether the data
|
||||
// should be copied on heap for off-heap allocators.
|
||||
@VisibleForTesting
|
||||
final MemtableTrie<BTreePartitionData> data;
|
||||
|
||||
private final ColumnsCollector columnsCollector;
|
||||
|
||||
private final StatsCollector statsCollector;
|
||||
|
||||
@Unmetered // total pool size should not be included in memtable's deep size
|
||||
private final MemtableAllocator allocator;
|
||||
|
||||
@Unmetered
|
||||
private final TrieMemtableMetricsView metrics;
|
||||
|
||||
@VisibleForTesting
|
||||
MemtableShard(TableMetadataRef metadata, MemtableAllocator allocator, TrieMemtableMetricsView metrics)
|
||||
{
|
||||
this.data = new MemtableTrie<>(BUFFER_TYPE);
|
||||
this.columnsCollector = new AbstractMemtable.ColumnsCollector(metadata.get().regularAndStaticColumns());
|
||||
this.statsCollector = new AbstractMemtable.StatsCollector();
|
||||
this.allocator = allocator;
|
||||
this.metrics = metrics;
|
||||
}
|
||||
|
||||
public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) throws MemtableTrie.SpaceExhaustedException
|
||||
{
|
||||
BTreePartitionUpdater updater = new BTreePartitionUpdater(allocator, allocator.cloner(opGroup), opGroup, indexer);
|
||||
boolean locked = writeLock.tryLock();
|
||||
if (locked)
|
||||
{
|
||||
metrics.uncontendedPuts.inc();
|
||||
}
|
||||
else
|
||||
{
|
||||
metrics.contendedPuts.inc();
|
||||
long lockStartTime = Clock.Global.nanoTime();
|
||||
writeLock.lock();
|
||||
metrics.contentionTime.addNano(Clock.Global.nanoTime() - lockStartTime);
|
||||
}
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
long onHeap = data.sizeOnHeap();
|
||||
long offHeap = data.sizeOffHeap();
|
||||
// Use the fast recursive put if we know the key is small enough to not cause a stack overflow.
|
||||
data.putSingleton(key,
|
||||
update,
|
||||
updater::mergePartitions,
|
||||
key.getKeyLength() < MAX_RECURSIVE_KEY_LENGTH);
|
||||
allocator.offHeap().adjust(data.sizeOffHeap() - offHeap, opGroup);
|
||||
allocator.onHeap().adjust(data.sizeOnHeap() - onHeap, opGroup);
|
||||
}
|
||||
finally
|
||||
{
|
||||
minTimestamp = Math.min(minTimestamp, update.stats().minTimestamp);
|
||||
minLocalDeletionTime = Math.min(minLocalDeletionTime, update.stats().minLocalDeletionTime);
|
||||
liveDataSize += updater.dataSize;
|
||||
currentOperations += update.operationCount();
|
||||
|
||||
columnsCollector.update(update.columns());
|
||||
statsCollector.update(update.stats());
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
writeLock.unlock();
|
||||
}
|
||||
return updater.colUpdateTimeDelta;
|
||||
}
|
||||
|
||||
public boolean isClean()
|
||||
{
|
||||
return data.isEmpty();
|
||||
}
|
||||
|
||||
public int size()
|
||||
{
|
||||
return data.valuesCount();
|
||||
}
|
||||
|
||||
long minTimestamp()
|
||||
{
|
||||
return minTimestamp;
|
||||
}
|
||||
|
||||
long liveDataSize()
|
||||
{
|
||||
return liveDataSize;
|
||||
}
|
||||
|
||||
long currentOperations()
|
||||
{
|
||||
return currentOperations;
|
||||
}
|
||||
|
||||
int minLocalDeletionTime()
|
||||
{
|
||||
return minLocalDeletionTime;
|
||||
}
|
||||
}
|
||||
|
||||
static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements UnfilteredPartitionIterator
|
||||
{
|
||||
private final TableMetadata metadata;
|
||||
private final EnsureOnHeap ensureOnHeap;
|
||||
private final Iterator<Map.Entry<ByteComparable, BTreePartitionData>> iter;
|
||||
private final ColumnFilter columnFilter;
|
||||
private final DataRange dataRange;
|
||||
|
||||
public MemtableUnfilteredPartitionIterator(TableMetadata metadata,
|
||||
EnsureOnHeap ensureOnHeap,
|
||||
Trie<BTreePartitionData> source,
|
||||
ColumnFilter columnFilter,
|
||||
DataRange dataRange)
|
||||
{
|
||||
this.metadata = metadata;
|
||||
this.ensureOnHeap = ensureOnHeap;
|
||||
this.iter = source.entryIterator();
|
||||
this.columnFilter = columnFilter;
|
||||
this.dataRange = dataRange;
|
||||
}
|
||||
|
||||
public TableMetadata metadata()
|
||||
{
|
||||
return metadata;
|
||||
}
|
||||
|
||||
public boolean hasNext()
|
||||
{
|
||||
return iter.hasNext();
|
||||
}
|
||||
|
||||
public UnfilteredRowIterator next()
|
||||
{
|
||||
Partition partition = getPartitionFromTrieEntry(metadata(), ensureOnHeap, iter.next());
|
||||
DecoratedKey key = partition.partitionKey();
|
||||
ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key);
|
||||
|
||||
return filter.getUnfilteredRowIterator(columnFilter, partition);
|
||||
}
|
||||
}
|
||||
|
||||
static class MemtablePartition extends ImmutableBTreePartition
|
||||
{
|
||||
|
||||
private final EnsureOnHeap ensureOnHeap;
|
||||
|
||||
private MemtablePartition(TableMetadata table, EnsureOnHeap ensureOnHeap, DecoratedKey key, BTreePartitionData data)
|
||||
{
|
||||
super(table, key, data);
|
||||
this.ensureOnHeap = ensureOnHeap;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean canHaveShadowedData()
|
||||
{
|
||||
// The BtreePartitionData we store in the memtable are build iteratively by BTreePartitionData.add(), which
|
||||
// doesn't make sure there isn't shadowed data, so we'll need to eliminate any.
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public DeletionInfo deletionInfo()
|
||||
{
|
||||
return ensureOnHeap.applyToDeletionInfo(super.deletionInfo());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Row staticRow()
|
||||
{
|
||||
return ensureOnHeap.applyToStatic(super.staticRow());
|
||||
}
|
||||
|
||||
@Override
|
||||
public DecoratedKey partitionKey()
|
||||
{
|
||||
return ensureOnHeap.applyToPartitionKey(super.partitionKey());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Row getRow(Clustering<?> clustering)
|
||||
{
|
||||
return ensureOnHeap.applyToRow(super.getRow(clustering));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Row lastRow()
|
||||
{
|
||||
return ensureOnHeap.applyToRow(super.lastRow());
|
||||
}
|
||||
|
||||
@Override
|
||||
public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, Slices slices, boolean reversed)
|
||||
{
|
||||
return unfilteredIterator(holder(), selection, slices, reversed);
|
||||
}
|
||||
|
||||
@Override
|
||||
public UnfilteredRowIterator unfilteredIterator(ColumnFilter selection, NavigableSet<Clustering<?>> clusteringsInQueryOrder, boolean reversed)
|
||||
{
|
||||
return ensureOnHeap.applyToPartition(super.unfilteredIterator(selection, clusteringsInQueryOrder, reversed));
|
||||
}
|
||||
|
||||
@Override
|
||||
public UnfilteredRowIterator unfilteredIterator()
|
||||
{
|
||||
return unfilteredIterator(ColumnFilter.selection(super.columns()), Slices.ALL, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public UnfilteredRowIterator unfilteredIterator(BTreePartitionData current, ColumnFilter selection, Slices slices, boolean reversed)
|
||||
{
|
||||
return ensureOnHeap.applyToPartition(super.unfilteredIterator(current, selection, slices, reversed));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<Row> iterator()
|
||||
{
|
||||
return ensureOnHeap.applyToPartition(super.iterator());
|
||||
}
|
||||
}
|
||||
|
||||
public static Factory factory(Map<String, String> optionsCopy)
|
||||
{
|
||||
String shardsString = optionsCopy.remove(SHARDS_OPTION);
|
||||
Integer shardCount = shardsString != null ? Integer.parseInt(shardsString) : null;
|
||||
return new Factory(shardCount);
|
||||
}
|
||||
|
||||
static class Factory implements Memtable.Factory
|
||||
{
|
||||
final Integer shardCount;
|
||||
|
||||
Factory(Integer shardCount)
|
||||
{
|
||||
this.shardCount = shardCount;
|
||||
}
|
||||
|
||||
public Memtable create(AtomicReference<CommitLogPosition> commitLogLowerBound,
|
||||
TableMetadataRef metadaRef,
|
||||
Owner owner)
|
||||
{
|
||||
return new TrieMemtable(commitLogLowerBound, metadaRef, owner, shardCount);
|
||||
}
|
||||
|
||||
@Override
|
||||
public TableMetrics.ReleasableMetric createMemtableMetrics(TableMetadataRef metadataRef)
|
||||
{
|
||||
TrieMemtableMetricsView metrics = new TrieMemtableMetricsView(metadataRef.keyspace, metadataRef.name);
|
||||
return metrics::release;
|
||||
}
|
||||
|
||||
public boolean equals(Object o)
|
||||
{
|
||||
if (this == o)
|
||||
return true;
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
Factory factory = (Factory) o;
|
||||
return Objects.equals(shardCount, factory.shardCount);
|
||||
}
|
||||
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(shardCount);
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public long unusedReservedMemory()
|
||||
{
|
||||
long size = 0;
|
||||
for (MemtableShard shard : shards)
|
||||
size += shard.data.unusedReservedMemory();
|
||||
return size;
|
||||
}
|
||||
}
|
||||
|
|
@ -22,14 +22,12 @@ import java.util.Collections;
|
|||
import java.util.Iterator;
|
||||
import java.util.NavigableSet;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.Iterators;
|
||||
|
||||
import org.apache.cassandra.schema.TableMetadata;
|
||||
import org.apache.cassandra.db.*;
|
||||
import org.apache.cassandra.db.filter.ColumnFilter;
|
||||
import org.apache.cassandra.db.rows.*;
|
||||
import org.apache.cassandra.utils.ObjectSizes;
|
||||
import org.apache.cassandra.utils.SearchIterator;
|
||||
import org.apache.cassandra.utils.btree.BTree;
|
||||
|
||||
|
|
@ -37,12 +35,9 @@ import static org.apache.cassandra.utils.btree.BTree.Dir.desc;
|
|||
|
||||
public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
||||
{
|
||||
protected static final Holder EMPTY = new Holder(RegularAndStaticColumns.NONE, BTree.empty(), DeletionInfo.LIVE, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
|
||||
public static final long HOLDER_UNSHARED_HEAP_SIZE = ObjectSizes.measure(EMPTY);
|
||||
|
||||
protected final DecoratedKey partitionKey;
|
||||
|
||||
protected abstract Holder holder();
|
||||
protected abstract BTreePartitionData holder();
|
||||
protected abstract boolean canHaveShadowedData();
|
||||
|
||||
protected AbstractBTreePartition(DecoratedKey partitionKey)
|
||||
|
|
@ -50,31 +45,6 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
this.partitionKey = partitionKey;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static final class Holder
|
||||
{
|
||||
public final RegularAndStaticColumns columns;
|
||||
public final DeletionInfo deletionInfo;
|
||||
// the btree of rows
|
||||
public final Object[] tree;
|
||||
public final Row staticRow;
|
||||
public final EncodingStats stats;
|
||||
|
||||
Holder(RegularAndStaticColumns columns, Object[] tree, DeletionInfo deletionInfo, Row staticRow, EncodingStats stats)
|
||||
{
|
||||
this.columns = columns;
|
||||
this.tree = tree;
|
||||
this.deletionInfo = deletionInfo;
|
||||
this.staticRow = staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow;
|
||||
this.stats = stats;
|
||||
}
|
||||
|
||||
protected Holder withColumns(RegularAndStaticColumns columns)
|
||||
{
|
||||
return new Holder(columns, this.tree, this.deletionInfo, this.staticRow, this.stats);
|
||||
}
|
||||
}
|
||||
|
||||
public DeletionInfo deletionInfo()
|
||||
{
|
||||
return holder().deletionInfo;
|
||||
|
|
@ -87,13 +57,13 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
|
||||
public boolean isEmpty()
|
||||
{
|
||||
Holder holder = holder();
|
||||
BTreePartitionData holder = holder();
|
||||
return holder.deletionInfo.isLive() && BTree.isEmpty(holder.tree) && holder.staticRow.isEmpty();
|
||||
}
|
||||
|
||||
public boolean hasRows()
|
||||
{
|
||||
Holder holder = holder();
|
||||
BTreePartitionData holder = holder();
|
||||
return !BTree.isEmpty(holder.tree);
|
||||
}
|
||||
|
||||
|
|
@ -122,7 +92,7 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
public Row getRow(Clustering<?> clustering)
|
||||
{
|
||||
ColumnFilter columns = ColumnFilter.selection(columns());
|
||||
Holder holder = holder();
|
||||
BTreePartitionData holder = holder();
|
||||
|
||||
if (clustering == Clustering.STATIC_CLUSTERING)
|
||||
{
|
||||
|
|
@ -152,7 +122,7 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
return row.filter(columns, activeDeletion, true, metadata());
|
||||
}
|
||||
|
||||
private Row staticRow(Holder current, ColumnFilter columns, boolean setActiveDeletionToRow)
|
||||
private Row staticRow(BTreePartitionData current, ColumnFilter columns, boolean setActiveDeletionToRow)
|
||||
{
|
||||
DeletionTime partitionDeletion = current.deletionInfo.getPartitionDeletion();
|
||||
if (columns.fetchedColumns().statics.isEmpty() || (current.staticRow.isEmpty() && partitionDeletion.isLive()))
|
||||
|
|
@ -185,7 +155,7 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
return unfilteredIterator(holder(), selection, slices, reversed);
|
||||
}
|
||||
|
||||
public UnfilteredRowIterator unfilteredIterator(Holder current, ColumnFilter selection, Slices slices, boolean reversed)
|
||||
public UnfilteredRowIterator unfilteredIterator(BTreePartitionData current, ColumnFilter selection, Slices slices, boolean reversed)
|
||||
{
|
||||
Row staticRow = staticRow(current, selection, false);
|
||||
if (slices.size() == 0)
|
||||
|
|
@ -199,7 +169,7 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
: new SlicesIterator(selection, slices, reversed, current, staticRow);
|
||||
}
|
||||
|
||||
private UnfilteredRowIterator sliceIterator(ColumnFilter selection, Slice slice, boolean reversed, Holder current, Row staticRow)
|
||||
private UnfilteredRowIterator sliceIterator(ColumnFilter selection, Slice slice, boolean reversed, BTreePartitionData current, Row staticRow)
|
||||
{
|
||||
ClusteringBound<?> start = slice.start().isBottom() ? null : slice.start();
|
||||
ClusteringBound<?> end = slice.end().isTop() ? null : slice.end();
|
||||
|
|
@ -209,7 +179,7 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
}
|
||||
|
||||
private RowAndDeletionMergeIterator merge(Iterator<Row> rowIter, Iterator<RangeTombstone> deleteIter,
|
||||
ColumnFilter selection, boolean reversed, Holder current, Row staticRow)
|
||||
ColumnFilter selection, boolean reversed, BTreePartitionData current, Row staticRow)
|
||||
{
|
||||
return new RowAndDeletionMergeIterator(metadata(), partitionKey(), current.deletionInfo.getPartitionDeletion(),
|
||||
selection, staticRow, reversed, current.stats,
|
||||
|
|
@ -219,10 +189,10 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
|
||||
private abstract class AbstractIterator extends AbstractUnfilteredRowIterator
|
||||
{
|
||||
final Holder current;
|
||||
final BTreePartitionData current;
|
||||
final ColumnFilter selection;
|
||||
|
||||
private AbstractIterator(Holder current, Row staticRow, ColumnFilter selection, boolean isReversed)
|
||||
private AbstractIterator(BTreePartitionData current, Row staticRow, ColumnFilter selection, boolean isReversed)
|
||||
{
|
||||
super(AbstractBTreePartition.this.metadata(),
|
||||
AbstractBTreePartition.this.partitionKey(),
|
||||
|
|
@ -245,7 +215,7 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
private int idx;
|
||||
private Iterator<Unfiltered> currentSlice;
|
||||
|
||||
private SlicesIterator(ColumnFilter selection, Slices slices, boolean isReversed, Holder current, Row staticRow)
|
||||
private SlicesIterator(ColumnFilter selection, Slices slices, boolean isReversed, BTreePartitionData current, Row staticRow)
|
||||
{
|
||||
super(current, staticRow, selection, isReversed);
|
||||
this.slices = slices;
|
||||
|
|
@ -283,7 +253,7 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
private ClusteringsIterator(ColumnFilter selection,
|
||||
NavigableSet<Clustering<?>> clusteringsInQueryOrder,
|
||||
boolean isReversed,
|
||||
Holder current,
|
||||
BTreePartitionData current,
|
||||
Row staticRow)
|
||||
{
|
||||
super(current, staticRow, selection, isReversed);
|
||||
|
|
@ -326,12 +296,12 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
}
|
||||
}
|
||||
|
||||
protected static Holder build(UnfilteredRowIterator iterator, int initialRowCapacity)
|
||||
protected static BTreePartitionData build(UnfilteredRowIterator iterator, int initialRowCapacity)
|
||||
{
|
||||
return build(iterator, initialRowCapacity, true);
|
||||
}
|
||||
|
||||
protected static Holder build(UnfilteredRowIterator iterator, int initialRowCapacity, boolean ordered)
|
||||
protected static BTreePartitionData build(UnfilteredRowIterator iterator, int initialRowCapacity, boolean ordered)
|
||||
{
|
||||
TableMetadata metadata = iterator.metadata();
|
||||
RegularAndStaticColumns columns = iterator.columns();
|
||||
|
|
@ -353,12 +323,12 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
if (reversed)
|
||||
builder.reverse();
|
||||
|
||||
return new Holder(columns, builder.build(), deletionBuilder.build(), iterator.staticRow(), iterator.stats());
|
||||
return new BTreePartitionData(columns, builder.build(), deletionBuilder.build(), iterator.staticRow(), iterator.stats());
|
||||
}
|
||||
|
||||
// Note that when building with a RowIterator, deletion will generally be LIVE, but we allow to pass it nonetheless because PartitionUpdate
|
||||
// passes a MutableDeletionInfo that it mutates later.
|
||||
protected static Holder build(RowIterator rows, DeletionInfo deletion, boolean buildEncodingStats)
|
||||
protected static BTreePartitionData build(RowIterator rows, DeletionInfo deletion, boolean buildEncodingStats)
|
||||
{
|
||||
RegularAndStaticColumns columns = rows.columns();
|
||||
boolean reversed = rows.isReverseOrder();
|
||||
|
|
@ -375,7 +345,7 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
Row staticRow = rows.staticRow();
|
||||
EncodingStats stats = buildEncodingStats ? EncodingStats.Collector.collect(staticRow, BTree.iterator(tree), deletion)
|
||||
: EncodingStats.NO_STATS;
|
||||
return new Holder(columns, tree, deletion, staticRow, stats);
|
||||
return new BTreePartitionData(columns, tree, deletion, staticRow, stats);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -420,7 +390,7 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
return false;
|
||||
|
||||
PartitionUpdate that = (PartitionUpdate) obj;
|
||||
Holder a = this.holder(), b = that.holder();
|
||||
BTreePartitionData a = this.holder(), b = that.holder();
|
||||
return partitionKey.equals(that.partitionKey)
|
||||
&& metadata().id.equals(that.metadata().id)
|
||||
&& a.deletionInfo.equals(b.deletionInfo)
|
||||
|
|
@ -446,16 +416,4 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row>
|
|||
|
||||
return BTree.findByIndex(tree, BTree.size(tree) - 1);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static Holder unsafeGetEmptyHolder()
|
||||
{
|
||||
return EMPTY;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static Holder unsafeConstructHolder(RegularAndStaticColumns columns, Object[] tree, DeletionInfo deletionInfo, Row staticRow, EncodingStats stats)
|
||||
{
|
||||
return new Holder(columns, tree, deletionInfo, staticRow, stats);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,29 +18,25 @@
|
|||
package org.apache.cassandra.db.partitions;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.NavigableSet;
|
||||
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
|
||||
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import org.apache.cassandra.index.transactions.UpdateTransaction;
|
||||
import org.apache.cassandra.schema.TableMetadata;
|
||||
import org.apache.cassandra.schema.TableMetadataRef;
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.db.*;
|
||||
import org.apache.cassandra.db.filter.ColumnFilter;
|
||||
import org.apache.cassandra.db.rows.*;
|
||||
import org.apache.cassandra.index.transactions.UpdateTransaction;
|
||||
import org.apache.cassandra.utils.ObjectSizes;
|
||||
import org.apache.cassandra.utils.btree.BTree;
|
||||
import org.apache.cassandra.utils.btree.UpdateFunction;
|
||||
import org.apache.cassandra.utils.concurrent.OpOrder;
|
||||
import org.apache.cassandra.utils.memory.Cloner;
|
||||
import org.apache.cassandra.utils.memory.HeapCloner;
|
||||
import org.apache.cassandra.utils.memory.MemtableAllocator;
|
||||
import org.github.jamm.Unmetered;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
|
||||
|
||||
|
|
@ -72,7 +68,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
|
|||
// CLOCK_GRANULARITY = 1^9ns >> CLOCK_SHIFT == 132us == (1/7.63)ms
|
||||
|
||||
private static final AtomicIntegerFieldUpdater<AtomicBTreePartition> wasteTrackerUpdater = AtomicIntegerFieldUpdater.newUpdater(AtomicBTreePartition.class, "wasteTracker");
|
||||
private static final AtomicReferenceFieldUpdater<AtomicBTreePartition, Holder> refUpdater = AtomicReferenceFieldUpdater.newUpdater(AtomicBTreePartition.class, Holder.class, "ref");
|
||||
private static final AtomicReferenceFieldUpdater<AtomicBTreePartition, BTreePartitionData> refUpdater = AtomicReferenceFieldUpdater.newUpdater(AtomicBTreePartition.class, BTreePartitionData.class, "ref");
|
||||
|
||||
/**
|
||||
* (clock + allocation) granularity are combined to give us an acceptable (waste) allocation rate that is defined by
|
||||
|
|
@ -87,7 +83,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
|
|||
@Unmetered
|
||||
private final MemtableAllocator allocator;
|
||||
|
||||
private volatile Holder ref;
|
||||
private volatile BTreePartitionData ref;
|
||||
|
||||
@Unmetered
|
||||
private final TableMetadataRef metadata;
|
||||
|
|
@ -98,10 +94,10 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
|
|||
super(partitionKey);
|
||||
this.metadata = metadata;
|
||||
this.allocator = allocator;
|
||||
this.ref = EMPTY;
|
||||
this.ref = BTreePartitionData.EMPTY;
|
||||
}
|
||||
|
||||
protected Holder holder()
|
||||
protected BTreePartitionData holder()
|
||||
{
|
||||
return ref;
|
||||
}
|
||||
|
|
@ -116,92 +112,78 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
|
|||
return true;
|
||||
}
|
||||
|
||||
private long[] addAllWithSizeDeltaInternal(RowUpdater updater, PartitionUpdate update, UpdateTransaction indexer)
|
||||
{
|
||||
Holder current = ref;
|
||||
updater.reset();
|
||||
|
||||
if (!update.deletionInfo().getPartitionDeletion().isLive())
|
||||
indexer.onPartitionDeletion(update.deletionInfo().getPartitionDeletion());
|
||||
|
||||
if (update.deletionInfo().hasRanges())
|
||||
update.deletionInfo().rangeIterator(false).forEachRemaining(indexer::onRangeTombstone);
|
||||
|
||||
DeletionInfo deletionInfo;
|
||||
if (update.deletionInfo().mayModify(current.deletionInfo))
|
||||
{
|
||||
if (updater.inputDeletionInfoCopy == null)
|
||||
updater.inputDeletionInfoCopy = update.deletionInfo().clone(HeapCloner.instance);
|
||||
|
||||
deletionInfo = current.deletionInfo.mutableCopy().add(updater.inputDeletionInfoCopy);
|
||||
updater.onAllocatedOnHeap(deletionInfo.unsharedHeapSize() - current.deletionInfo.unsharedHeapSize());
|
||||
}
|
||||
else
|
||||
{
|
||||
deletionInfo = current.deletionInfo;
|
||||
}
|
||||
|
||||
RegularAndStaticColumns columns = update.columns().mergeTo(current.columns);
|
||||
updater.onAllocatedOnHeap(columns.unsharedHeapSize() - current.columns.unsharedHeapSize());
|
||||
Row newStatic = update.staticRow();
|
||||
Row staticRow = newStatic.isEmpty()
|
||||
? current.staticRow
|
||||
: (current.staticRow.isEmpty() ? updater.insert(newStatic) : updater.merge(current.staticRow, newStatic));
|
||||
Object[] tree = BTree.update(current.tree, update.holder().tree, update.metadata().comparator, updater);
|
||||
EncodingStats newStats = current.stats.mergeWith(update.stats());
|
||||
updater.onAllocatedOnHeap(newStats.unsharedHeapSize() - current.stats.unsharedHeapSize());
|
||||
|
||||
if (tree != null && refUpdater.compareAndSet(this, current, new Holder(columns, tree, deletionInfo, staticRow, newStats)))
|
||||
{
|
||||
updater.finish();
|
||||
return new long[]{ updater.dataSize, updater.colUpdateTimeDelta };
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Adds a given update to this in-memtable partition.
|
||||
*
|
||||
* @return an array containing first the difference in size seen after merging the updates, and second the minimum
|
||||
* time detla between updates.
|
||||
* time delta between updates.
|
||||
*/
|
||||
public long[] addAllWithSizeDelta(final PartitionUpdate update,
|
||||
Cloner cloner,
|
||||
OpOrder.Group writeOp,
|
||||
UpdateTransaction indexer)
|
||||
public BTreePartitionUpdater addAll(final PartitionUpdate update, Cloner cloner, OpOrder.Group writeOp, UpdateTransaction indexer)
|
||||
{
|
||||
RowUpdater updater = new RowUpdater(allocator, cloner, writeOp, indexer);
|
||||
try
|
||||
{
|
||||
boolean shouldLock = shouldLock(writeOp);
|
||||
indexer.start();
|
||||
return new Updater(allocator, cloner, writeOp, indexer).addAll(update);
|
||||
}
|
||||
|
||||
while (true)
|
||||
@VisibleForTesting
|
||||
public void unsafeSetHolder(BTreePartitionData holder)
|
||||
{
|
||||
ref = holder;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public BTreePartitionData unsafeGetHolder()
|
||||
{
|
||||
return ref;
|
||||
}
|
||||
|
||||
class Updater extends BTreePartitionUpdater
|
||||
{
|
||||
BTreePartitionData current;
|
||||
|
||||
public Updater(MemtableAllocator allocator, Cloner cloner, OpOrder.Group writeOp, UpdateTransaction indexer)
|
||||
{
|
||||
super(allocator, cloner, writeOp, indexer);
|
||||
}
|
||||
|
||||
Updater addAll(final PartitionUpdate update)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (shouldLock)
|
||||
boolean shouldLock = shouldLock(writeOp);
|
||||
indexer.start();
|
||||
|
||||
while (true)
|
||||
{
|
||||
synchronized (this)
|
||||
if (shouldLock)
|
||||
{
|
||||
long[] result = addAllWithSizeDeltaInternal(updater, update, indexer);
|
||||
if (result != null)
|
||||
return result;
|
||||
synchronized (this)
|
||||
{
|
||||
if (tryUpdateData(update))
|
||||
return this;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (tryUpdateData(update))
|
||||
return this;
|
||||
|
||||
shouldLock = shouldLock(heapSize, writeOp);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
long[] result = addAllWithSizeDeltaInternal(updater, update, indexer);
|
||||
if (result != null)
|
||||
return result;
|
||||
|
||||
shouldLock = shouldLock(updater.heapSize, writeOp);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
indexer.commit();
|
||||
reportAllocatedMemory();
|
||||
}
|
||||
}
|
||||
finally
|
||||
|
||||
private boolean tryUpdateData(PartitionUpdate update)
|
||||
{
|
||||
indexer.commit();
|
||||
current = ref;
|
||||
this.dataSize = 0;
|
||||
this.heapSize = 0;
|
||||
BTreePartitionData result = makeMergedPartition(current, update);
|
||||
return refUpdater.compareAndSet(AtomicBTreePartition.this, current, result);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -254,7 +236,7 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
|
|||
}
|
||||
|
||||
@Override
|
||||
public UnfilteredRowIterator unfilteredIterator(Holder current, ColumnFilter selection, Slices slices, boolean reversed)
|
||||
public UnfilteredRowIterator unfilteredIterator(BTreePartitionData current, ColumnFilter selection, Slices slices, boolean reversed)
|
||||
{
|
||||
return allocator.ensureOnHeap().applyToPartition(super.unfilteredIterator(current, selection, slices, reversed));
|
||||
}
|
||||
|
|
@ -338,119 +320,4 @@ public final class AtomicBTreePartition extends AbstractBTreePartition
|
|||
return wasteTracker + 1;
|
||||
return wasteTracker;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public void unsafeSetHolder(Holder holder)
|
||||
{
|
||||
ref = holder;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public Holder unsafeGetHolder()
|
||||
{
|
||||
return ref;
|
||||
}
|
||||
|
||||
// the function we provide to the btree utilities to perform any column replacements
|
||||
private static final class RowUpdater implements UpdateFunction<Row, Row>, ColumnData.PostReconciliationFunction
|
||||
{
|
||||
final MemtableAllocator allocator;
|
||||
final OpOrder.Group writeOp;
|
||||
final UpdateTransaction indexer;
|
||||
final Cloner cloner;
|
||||
long dataSize;
|
||||
long heapSize;
|
||||
long colUpdateTimeDelta = Long.MAX_VALUE;
|
||||
List<Row> inserted; // TODO: replace with walk of aborted BTree
|
||||
|
||||
DeletionInfo inputDeletionInfoCopy = null;
|
||||
|
||||
private RowUpdater(MemtableAllocator allocator, Cloner cloner, OpOrder.Group writeOp, UpdateTransaction indexer)
|
||||
{
|
||||
this.allocator = allocator;
|
||||
this.writeOp = writeOp;
|
||||
this.indexer = indexer;
|
||||
this.cloner = cloner;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Row insert(Row insert)
|
||||
{
|
||||
Row data = insert.clone(cloner);
|
||||
indexer.onInserted(insert);
|
||||
|
||||
this.dataSize += data.dataSize();
|
||||
onAllocatedOnHeap(data.unsharedHeapSizeExcludingData());
|
||||
if (inserted == null)
|
||||
inserted = new ArrayList<>();
|
||||
inserted.add(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
public Row merge(Row existing, Row update)
|
||||
{
|
||||
Row reconciled = Rows.merge(existing, update, this);
|
||||
indexer.onUpdated(existing, reconciled);
|
||||
|
||||
if (inserted == null)
|
||||
inserted = new ArrayList<>();
|
||||
inserted.add(reconciled);
|
||||
|
||||
return reconciled;
|
||||
}
|
||||
|
||||
public Row retain(Row existing)
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
|
||||
protected void reset()
|
||||
{
|
||||
this.dataSize = 0;
|
||||
this.heapSize = 0;
|
||||
if (inserted != null)
|
||||
inserted.clear();
|
||||
}
|
||||
|
||||
public Cell<?> merge(Cell<?> previous, Cell<?> insert)
|
||||
{
|
||||
if (insert != previous)
|
||||
{
|
||||
long timeDelta = Math.abs(insert.timestamp() - previous.timestamp());
|
||||
if (timeDelta < colUpdateTimeDelta)
|
||||
colUpdateTimeDelta = timeDelta;
|
||||
}
|
||||
if (cloner != null)
|
||||
insert = cloner.clone(insert);
|
||||
dataSize += insert.dataSize() - previous.dataSize();
|
||||
heapSize += insert.unsharedHeapSizeExcludingData() - previous.unsharedHeapSizeExcludingData();
|
||||
return insert;
|
||||
}
|
||||
|
||||
public ColumnData insert(ColumnData insert)
|
||||
{
|
||||
if (cloner != null)
|
||||
insert = insert.clone(cloner);
|
||||
dataSize += insert.dataSize();
|
||||
heapSize += insert.unsharedHeapSizeExcludingData();
|
||||
return insert;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(ColumnData existing)
|
||||
{
|
||||
dataSize -= existing.dataSize();
|
||||
heapSize -= existing.unsharedHeapSizeExcludingData();
|
||||
}
|
||||
|
||||
public void onAllocatedOnHeap(long heapSize)
|
||||
{
|
||||
this.heapSize += heapSize;
|
||||
}
|
||||
|
||||
protected void finish()
|
||||
{
|
||||
allocator.onHeap().adjust(heapSize, writeOp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.db.partitions;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import org.apache.cassandra.db.DeletionInfo;
|
||||
import org.apache.cassandra.db.RegularAndStaticColumns;
|
||||
import org.apache.cassandra.db.rows.EncodingStats;
|
||||
import org.apache.cassandra.db.rows.Row;
|
||||
import org.apache.cassandra.db.rows.Rows;
|
||||
import org.apache.cassandra.utils.ObjectSizes;
|
||||
import org.apache.cassandra.utils.btree.BTree;
|
||||
|
||||
/**
|
||||
* Holder of the content of a partition, see {@link AbstractBTreePartition}.
|
||||
* When updating a partition one holder is swapped for another atomically.
|
||||
*/
|
||||
public final class BTreePartitionData
|
||||
{
|
||||
public static final BTreePartitionData EMPTY = new BTreePartitionData(RegularAndStaticColumns.NONE,
|
||||
BTree.empty(),
|
||||
DeletionInfo.LIVE,
|
||||
Rows.EMPTY_STATIC_ROW,
|
||||
EncodingStats.NO_STATS);
|
||||
public static final long UNSHARED_HEAP_SIZE = ObjectSizes.measure(EMPTY);
|
||||
|
||||
|
||||
final RegularAndStaticColumns columns;
|
||||
final DeletionInfo deletionInfo;
|
||||
// the btree of rows
|
||||
final Object[] tree;
|
||||
final Row staticRow;
|
||||
public final EncodingStats stats;
|
||||
|
||||
BTreePartitionData(RegularAndStaticColumns columns,
|
||||
Object[] tree,
|
||||
DeletionInfo deletionInfo,
|
||||
Row staticRow,
|
||||
EncodingStats stats)
|
||||
{
|
||||
this.columns = columns;
|
||||
this.tree = tree;
|
||||
this.deletionInfo = deletionInfo;
|
||||
this.staticRow = staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow;
|
||||
this.stats = stats;
|
||||
}
|
||||
|
||||
BTreePartitionData withColumns(RegularAndStaticColumns columns)
|
||||
{
|
||||
return new BTreePartitionData(columns, this.tree, this.deletionInfo, this.staticRow, this.stats);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static BTreePartitionData unsafeGetEmpty()
|
||||
{
|
||||
return EMPTY;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static BTreePartitionData unsafeConstruct(RegularAndStaticColumns columns,
|
||||
Object[] tree,
|
||||
DeletionInfo deletionInfo,
|
||||
Row staticRow,
|
||||
EncodingStats stats)
|
||||
{
|
||||
return new BTreePartitionData(columns, tree, deletionInfo, staticRow, stats);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static void unsafeInvalidate(AtomicBTreePartition partition)
|
||||
{
|
||||
BTreePartitionData holder = partition.unsafeGetHolder();
|
||||
if (!BTree.isEmpty(holder.tree))
|
||||
{
|
||||
partition.unsafeSetHolder(unsafeConstruct(holder.columns,
|
||||
Arrays.copyOf(holder.tree, holder.tree.length),
|
||||
holder.deletionInfo,
|
||||
holder.staticRow,
|
||||
holder.stats));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.db.partitions;
|
||||
|
||||
import org.apache.cassandra.db.DeletionInfo;
|
||||
import org.apache.cassandra.db.RegularAndStaticColumns;
|
||||
import org.apache.cassandra.db.rows.Cell;
|
||||
import org.apache.cassandra.db.rows.ColumnData;
|
||||
import org.apache.cassandra.db.rows.EncodingStats;
|
||||
import org.apache.cassandra.db.rows.Row;
|
||||
import org.apache.cassandra.db.rows.Rows;
|
||||
import org.apache.cassandra.index.transactions.UpdateTransaction;
|
||||
import org.apache.cassandra.utils.btree.BTree;
|
||||
import org.apache.cassandra.utils.btree.UpdateFunction;
|
||||
import org.apache.cassandra.utils.concurrent.OpOrder;
|
||||
import org.apache.cassandra.utils.memory.Cloner;
|
||||
import org.apache.cassandra.utils.memory.HeapCloner;
|
||||
import org.apache.cassandra.utils.memory.MemtableAllocator;
|
||||
|
||||
/**
|
||||
* the function we provide to the trie and btree utilities to perform any row and column replacements
|
||||
*/
|
||||
public class BTreePartitionUpdater implements UpdateFunction<Row, Row>, ColumnData.PostReconciliationFunction
|
||||
{
|
||||
final MemtableAllocator allocator;
|
||||
final OpOrder.Group writeOp;
|
||||
final Cloner cloner;
|
||||
final UpdateTransaction indexer;
|
||||
public long dataSize;
|
||||
long heapSize;
|
||||
public long colUpdateTimeDelta = Long.MAX_VALUE;
|
||||
|
||||
public BTreePartitionUpdater(MemtableAllocator allocator, Cloner cloner, OpOrder.Group writeOp, UpdateTransaction indexer)
|
||||
{
|
||||
this.allocator = allocator;
|
||||
this.cloner = cloner;
|
||||
this.writeOp = writeOp;
|
||||
this.indexer = indexer;
|
||||
this.heapSize = 0;
|
||||
this.dataSize = 0;
|
||||
}
|
||||
|
||||
public BTreePartitionData mergePartitions(BTreePartitionData current, final PartitionUpdate update)
|
||||
{
|
||||
if (current == null)
|
||||
{
|
||||
current = BTreePartitionData.EMPTY;
|
||||
onAllocatedOnHeap(BTreePartitionData.UNSHARED_HEAP_SIZE);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
indexer.start();
|
||||
|
||||
return makeMergedPartition(current, update);
|
||||
}
|
||||
finally
|
||||
{
|
||||
indexer.commit();
|
||||
reportAllocatedMemory();
|
||||
}
|
||||
}
|
||||
|
||||
protected BTreePartitionData makeMergedPartition(BTreePartitionData current, PartitionUpdate update)
|
||||
{
|
||||
DeletionInfo newDeletionInfo = merge(current.deletionInfo, update.deletionInfo());
|
||||
|
||||
RegularAndStaticColumns columns = current.columns;
|
||||
RegularAndStaticColumns newColumns = update.columns().mergeTo(columns);
|
||||
onAllocatedOnHeap(newColumns.unsharedHeapSize() - columns.unsharedHeapSize());
|
||||
Row newStatic = mergeStatic(current.staticRow, update.staticRow());
|
||||
|
||||
Object[] tree = BTree.update(current.tree, update.holder().tree, update.metadata().comparator, this);
|
||||
EncodingStats newStats = current.stats.mergeWith(update.stats());
|
||||
onAllocatedOnHeap(newStats.unsharedHeapSize() - current.stats.unsharedHeapSize());
|
||||
|
||||
return new BTreePartitionData(newColumns, tree, newDeletionInfo, newStatic, newStats);
|
||||
}
|
||||
|
||||
private Row mergeStatic(Row current, Row update)
|
||||
{
|
||||
if (update.isEmpty())
|
||||
return current;
|
||||
if (current.isEmpty())
|
||||
return insert(update);
|
||||
|
||||
return merge(current, update);
|
||||
}
|
||||
|
||||
private DeletionInfo merge(DeletionInfo existing, DeletionInfo update)
|
||||
{
|
||||
if (update.isLive() || !update.mayModify(existing))
|
||||
return existing;
|
||||
|
||||
if (!update.getPartitionDeletion().isLive())
|
||||
indexer.onPartitionDeletion(update.getPartitionDeletion());
|
||||
|
||||
if (update.hasRanges())
|
||||
update.rangeIterator(false).forEachRemaining(indexer::onRangeTombstone);
|
||||
|
||||
// Like for rows, we have to clone the update in case internal buffers (when it has range tombstones) reference
|
||||
// memory we shouldn't hold into. But we don't ever store this off-heap currently so we just default to the
|
||||
// HeapAllocator (rather than using 'allocator').
|
||||
DeletionInfo newInfo = existing.mutableCopy().add(update.clone(HeapCloner.instance));
|
||||
onAllocatedOnHeap(newInfo.unsharedHeapSize() - existing.unsharedHeapSize());
|
||||
return newInfo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Row insert(Row insert)
|
||||
{
|
||||
Row data = insert.clone(cloner);
|
||||
indexer.onInserted(insert);
|
||||
|
||||
dataSize += data.dataSize();
|
||||
onAllocatedOnHeap(data.unsharedHeapSizeExcludingData());
|
||||
return data;
|
||||
}
|
||||
|
||||
public Row merge(Row existing, Row update)
|
||||
{
|
||||
Row reconciled = Rows.merge(existing, update, this);
|
||||
indexer.onUpdated(existing, reconciled);
|
||||
|
||||
return reconciled;
|
||||
}
|
||||
|
||||
public Cell<?> merge(Cell<?> previous, Cell<?> insert)
|
||||
{
|
||||
if (insert != previous)
|
||||
{
|
||||
long timeDelta = Math.abs(insert.timestamp() - previous.timestamp());
|
||||
if (timeDelta < colUpdateTimeDelta)
|
||||
colUpdateTimeDelta = timeDelta;
|
||||
}
|
||||
if (cloner != null)
|
||||
insert = cloner.clone(insert);
|
||||
dataSize += insert.dataSize() - previous.dataSize();
|
||||
heapSize += insert.unsharedHeapSizeExcludingData() - previous.unsharedHeapSizeExcludingData();
|
||||
return insert;
|
||||
}
|
||||
|
||||
public ColumnData insert(ColumnData insert)
|
||||
{
|
||||
if (cloner != null)
|
||||
insert = insert.clone(cloner);
|
||||
dataSize += insert.dataSize();
|
||||
heapSize += insert.unsharedHeapSizeExcludingData();
|
||||
return insert;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(ColumnData existing)
|
||||
{
|
||||
dataSize -= existing.dataSize();
|
||||
heapSize -= existing.unsharedHeapSizeExcludingData();
|
||||
}
|
||||
|
||||
public void onAllocatedOnHeap(long heapSize)
|
||||
{
|
||||
this.heapSize += heapSize;
|
||||
}
|
||||
|
||||
public void reportAllocatedMemory()
|
||||
{
|
||||
allocator.onHeap().adjust(heapSize, writeOp);
|
||||
}
|
||||
}
|
||||
|
|
@ -40,7 +40,7 @@ public class CachedBTreePartition extends ImmutableBTreePartition implements Cac
|
|||
|
||||
private CachedBTreePartition(TableMetadata metadata,
|
||||
DecoratedKey partitionKey,
|
||||
Holder holder,
|
||||
BTreePartitionData holder,
|
||||
int createdAtInSec,
|
||||
int cachedLiveRows,
|
||||
int rowsWithNonExpiringCells)
|
||||
|
|
@ -80,7 +80,7 @@ public class CachedBTreePartition extends ImmutableBTreePartition implements Cac
|
|||
*/
|
||||
public static CachedBTreePartition create(UnfilteredRowIterator iterator, int initialRowCapacity, int nowInSec)
|
||||
{
|
||||
Holder holder = ImmutableBTreePartition.build(iterator, initialRowCapacity);
|
||||
BTreePartitionData holder = ImmutableBTreePartition.build(iterator, initialRowCapacity);
|
||||
|
||||
int cachedLiveRows = 0;
|
||||
int rowsWithNonExpiringCells = 0;
|
||||
|
|
@ -180,7 +180,7 @@ public class CachedBTreePartition extends ImmutableBTreePartition implements Cac
|
|||
UnfilteredRowIteratorSerializer.Header header = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(metadata, null, in, version, DeserializationHelper.Flag.LOCAL);
|
||||
assert !header.isReversed && header.rowEstimate >= 0;
|
||||
|
||||
Holder holder;
|
||||
BTreePartitionData holder;
|
||||
try (UnfilteredRowIterator partition = UnfilteredRowIteratorSerializer.serializer.deserialize(in, version, metadata, DeserializationHelper.Flag.LOCAL, header))
|
||||
{
|
||||
holder = ImmutableBTreePartition.build(partition, header.rowEstimate);
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ import org.apache.cassandra.db.rows.*;
|
|||
public class ImmutableBTreePartition extends AbstractBTreePartition
|
||||
{
|
||||
|
||||
protected final Holder holder;
|
||||
protected final BTreePartitionData holder;
|
||||
protected final TableMetadata metadata;
|
||||
|
||||
public ImmutableBTreePartition(TableMetadata metadata,
|
||||
|
|
@ -40,12 +40,12 @@ public class ImmutableBTreePartition extends AbstractBTreePartition
|
|||
{
|
||||
super(partitionKey);
|
||||
this.metadata = metadata;
|
||||
this.holder = new Holder(columns, tree, deletionInfo, staticRow, stats);
|
||||
this.holder = new BTreePartitionData(columns, tree, deletionInfo, staticRow, stats);
|
||||
}
|
||||
|
||||
protected ImmutableBTreePartition(TableMetadata metadata,
|
||||
DecoratedKey partitionKey,
|
||||
Holder holder)
|
||||
BTreePartitionData holder)
|
||||
{
|
||||
super(partitionKey);
|
||||
this.metadata = metadata;
|
||||
|
|
@ -119,7 +119,7 @@ public class ImmutableBTreePartition extends AbstractBTreePartition
|
|||
return metadata;
|
||||
}
|
||||
|
||||
protected Holder holder()
|
||||
protected BTreePartitionData holder()
|
||||
{
|
||||
return holder;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
|
||||
public static final PartitionUpdateSerializer serializer = new PartitionUpdateSerializer();
|
||||
|
||||
private final Holder holder;
|
||||
private final BTreePartitionData holder;
|
||||
private final DeletionInfo deletionInfo;
|
||||
private final TableMetadata metadata;
|
||||
|
||||
|
|
@ -75,7 +75,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
|
||||
private PartitionUpdate(TableMetadata metadata,
|
||||
DecoratedKey key,
|
||||
Holder holder,
|
||||
BTreePartitionData holder,
|
||||
MutableDeletionInfo deletionInfo,
|
||||
boolean canHaveShadowedData)
|
||||
{
|
||||
|
|
@ -97,7 +97,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
public static PartitionUpdate emptyUpdate(TableMetadata metadata, DecoratedKey key)
|
||||
{
|
||||
MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
|
||||
Holder holder = new Holder(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
|
||||
BTreePartitionData holder = new BTreePartitionData(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
|
||||
return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
|
||||
}
|
||||
|
||||
|
|
@ -114,7 +114,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
public static PartitionUpdate fullPartitionDelete(TableMetadata metadata, DecoratedKey key, long timestamp, int nowInSec)
|
||||
{
|
||||
MutableDeletionInfo deletionInfo = new MutableDeletionInfo(timestamp, nowInSec);
|
||||
Holder holder = new Holder(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
|
||||
BTreePartitionData holder = new BTreePartitionData(RegularAndStaticColumns.NONE, BTree.empty(), deletionInfo, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS);
|
||||
return new PartitionUpdate(metadata, key, holder, deletionInfo, false);
|
||||
}
|
||||
|
||||
|
|
@ -131,7 +131,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
public static PartitionUpdate singleRowUpdate(TableMetadata metadata, DecoratedKey key, Row row, Row staticRow)
|
||||
{
|
||||
MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
|
||||
Holder holder = new Holder(
|
||||
BTreePartitionData holder = new BTreePartitionData(
|
||||
new RegularAndStaticColumns(
|
||||
staticRow == null ? Columns.NONE : Columns.from(staticRow),
|
||||
row == null ? Columns.NONE : Columns.from(row)
|
||||
|
|
@ -187,7 +187,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
public static PartitionUpdate fromIterator(UnfilteredRowIterator iterator, ColumnFilter filter)
|
||||
{
|
||||
iterator = UnfilteredRowIterators.withOnlyQueriedData(iterator, filter);
|
||||
Holder holder = build(iterator, 16);
|
||||
BTreePartitionData holder = build(iterator, 16);
|
||||
MutableDeletionInfo deletionInfo = (MutableDeletionInfo) holder.deletionInfo;
|
||||
return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
|
||||
}
|
||||
|
|
@ -208,7 +208,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
{
|
||||
iterator = RowIterators.withOnlyQueriedData(iterator, filter);
|
||||
MutableDeletionInfo deletionInfo = MutableDeletionInfo.live();
|
||||
Holder holder = build(iterator, deletionInfo, true);
|
||||
BTreePartitionData holder = build(iterator, deletionInfo, true);
|
||||
return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false);
|
||||
}
|
||||
|
||||
|
|
@ -373,7 +373,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
return holder.columns;
|
||||
}
|
||||
|
||||
protected Holder holder()
|
||||
protected BTreePartitionData holder()
|
||||
{
|
||||
return holder;
|
||||
}
|
||||
|
|
@ -494,7 +494,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
@VisibleForTesting
|
||||
public static PartitionUpdate unsafeConstruct(TableMetadata metadata,
|
||||
DecoratedKey key,
|
||||
Holder holder,
|
||||
BTreePartitionData holder,
|
||||
MutableDeletionInfo deletionInfo,
|
||||
boolean canHaveShadowedData)
|
||||
{
|
||||
|
|
@ -690,7 +690,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
MutableDeletionInfo deletionInfo = deletionBuilder.build();
|
||||
return new PartitionUpdate(metadata,
|
||||
header.key,
|
||||
new Holder(header.sHeader.columns(), rows, deletionInfo, header.staticRow, header.sHeader.stats()),
|
||||
new BTreePartitionData(header.sHeader.columns(), rows, deletionInfo, header.staticRow, header.sHeader.stats()),
|
||||
deletionInfo,
|
||||
false);
|
||||
}
|
||||
|
|
@ -800,7 +800,7 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
RegularAndStaticColumns columns,
|
||||
int initialRowCapacity,
|
||||
boolean canHaveShadowedData,
|
||||
Holder holder)
|
||||
BTreePartitionData holder)
|
||||
{
|
||||
this(metadata, key, columns, initialRowCapacity, canHaveShadowedData, holder.staticRow, holder.deletionInfo, holder.tree);
|
||||
}
|
||||
|
|
@ -909,11 +909,11 @@ public class PartitionUpdate extends AbstractBTreePartition
|
|||
isBuilt = true;
|
||||
return new PartitionUpdate(metadata,
|
||||
partitionKey(),
|
||||
new Holder(columns,
|
||||
merged,
|
||||
deletionInfo,
|
||||
staticRow,
|
||||
newStats),
|
||||
new BTreePartitionData(columns,
|
||||
merged,
|
||||
deletionInfo,
|
||||
staticRow,
|
||||
newStats),
|
||||
deletionInfo,
|
||||
canHaveShadowedData);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -176,9 +176,6 @@ public class MemtableReadTrie<T> extends Trie<T>
|
|||
// Offset of the next pointer in a non-shared prefix node
|
||||
static final int PREFIX_POINTER_OFFSET = LAST_POINTER_OFFSET - PREFIX_OFFSET;
|
||||
|
||||
// Initial capacity for the node data buffer.
|
||||
static final int INITIAL_BUFFER_CAPACITY = 256;
|
||||
|
||||
/**
|
||||
* Value used as null for node pointers.
|
||||
* No node can use this address (we enforce this by not allowing chain nodes to grow to position 0).
|
||||
|
|
@ -214,6 +211,11 @@ public class MemtableReadTrie<T> extends Trie<T>
|
|||
static final int CONTENTS_START_SHIFT = 4;
|
||||
static final int CONTENTS_START_SIZE = 1 << CONTENTS_START_SHIFT;
|
||||
|
||||
static
|
||||
{
|
||||
assert BUF_START_SIZE % BLOCK_SIZE == 0 : "Initial buffer size must fit a full block.";
|
||||
}
|
||||
|
||||
final UnsafeBuffer[] buffers;
|
||||
final AtomicReferenceArray<T>[] contentArrays;
|
||||
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ public class MemtableTrie<T> extends MemtableReadTrie<T>
|
|||
// constants for space calculations
|
||||
private static final long EMPTY_SIZE_ON_HEAP;
|
||||
private static final long EMPTY_SIZE_OFF_HEAP;
|
||||
private static final long REFERENCE_ARRAY_ON_HEAP_SIZE = ObjectSizes.measureDeep(new AtomicReferenceArray(0));
|
||||
private static final long REFERENCE_ARRAY_ON_HEAP_SIZE = ObjectSizes.measureDeep(new AtomicReferenceArray<>(0));
|
||||
|
||||
static
|
||||
{
|
||||
|
|
@ -91,7 +91,6 @@ public class MemtableTrie<T> extends MemtableReadTrie<T>
|
|||
new AtomicReferenceArray[29 - CONTENTS_START_SHIFT], // takes at least 4 bytes to write pointer to one content -> 4 times smaller than buffers
|
||||
NONE);
|
||||
this.bufferType = bufferType;
|
||||
assert INITIAL_BUFFER_CAPACITY % BLOCK_SIZE == 0;
|
||||
}
|
||||
|
||||
// Buffer, content list and block management
|
||||
|
|
@ -117,11 +116,6 @@ public class MemtableTrie<T> extends MemtableReadTrie<T>
|
|||
getChunk(pos).putInt(inChunkPointer(pos), value);
|
||||
}
|
||||
|
||||
final void putIntOrdered(int pos, int value)
|
||||
{
|
||||
getChunk(pos).putIntOrdered(inChunkPointer(pos), value);
|
||||
}
|
||||
|
||||
final void putIntVolatile(int pos, int value)
|
||||
{
|
||||
getChunk(pos).putIntVolatile(inChunkPointer(pos), value);
|
||||
|
|
@ -149,20 +143,16 @@ public class MemtableTrie<T> extends MemtableReadTrie<T>
|
|||
// close to the 2G limit.
|
||||
int v = allocatedPos;
|
||||
if (inChunkPointer(v) == 0)
|
||||
{
|
||||
{
|
||||
int leadBit = getChunkIdx(v, BUF_START_SHIFT, BUF_START_SIZE);
|
||||
if (leadBit == 31)
|
||||
throw new SpaceExhaustedException();
|
||||
if (leadBit + BUF_START_SHIFT == 31)
|
||||
throw new SpaceExhaustedException();
|
||||
|
||||
assert buffers[leadBit] == null;
|
||||
ByteBuffer newBuffer = bufferType.allocate(BUF_START_SIZE << leadBit);
|
||||
buffers[leadBit] = new UnsafeBuffer(newBuffer);
|
||||
// The above does not contain any happens-before enforcing writes, thus at this point the new buffer may be
|
||||
// invisible to any concurrent readers. Touching the volatile root pointer (which any new read must go
|
||||
// through) enforces a happens-before that makes it visible to all new reads (note: when the write completes
|
||||
// it must do some volatile write, but that will be in the new buffer and without the line below could
|
||||
// remain unreachable by other cores).
|
||||
root = root;
|
||||
// Note: Since we are not moving existing data to a new buffer, we are okay with no happens-before enforcing
|
||||
// writes. Any reader that sees a pointer in the new buffer may only do so after reading the volatile write
|
||||
// that attached the new path.
|
||||
}
|
||||
|
||||
allocatedPos += BLOCK_SIZE;
|
||||
|
|
@ -177,11 +167,11 @@ public class MemtableTrie<T> extends MemtableReadTrie<T>
|
|||
AtomicReferenceArray<T> array = contentArrays[leadBit];
|
||||
if (array == null)
|
||||
{
|
||||
assert ofs == 0;
|
||||
assert ofs == 0 : "Error in content arrays configuration.";
|
||||
contentArrays[leadBit] = array = new AtomicReferenceArray<>(CONTENTS_START_SIZE << leadBit);
|
||||
}
|
||||
array.lazySet(ofs, value); // no need for a volatile set here; at this point the item is not referenced
|
||||
// by any node in the trie, and a volatile set will be made to reference it.
|
||||
// by any node in the trie, and a volatile set will be made to reference it.
|
||||
return index;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -118,7 +118,6 @@ class MergeTrie<T> extends Trie<T>
|
|||
int c2trans = c2.incomingTransition();
|
||||
atC1 = c1trans <= c2trans;
|
||||
atC2 = c1trans >= c2trans;
|
||||
assert atC1 | atC2;
|
||||
return c1depth;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -253,8 +253,8 @@ public class MetadataCollector implements PartitionStatisticsCollector
|
|||
|
||||
public MetadataCollector updateClusteringValues(ClusteringPrefix<?> clustering)
|
||||
{
|
||||
minClustering = minClustering == null || comparator.compare(clustering, minClustering) < 0 ? clustering.minimize() : minClustering;
|
||||
maxClustering = maxClustering == null || comparator.compare(clustering, maxClustering) > 0 ? clustering.minimize() : maxClustering;
|
||||
minClustering = minClustering == null || comparator.compare(clustering, minClustering) < 0 ? clustering : minClustering;
|
||||
maxClustering = maxClustering == null || comparator.compare(clustering, maxClustering) > 0 ? clustering : maxClustering;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
|
@ -267,8 +267,8 @@ public class MetadataCollector implements PartitionStatisticsCollector
|
|||
{
|
||||
Preconditions.checkState((minClustering == null && maxClustering == null)
|
||||
|| comparator.compare(maxClustering, minClustering) >= 0);
|
||||
ByteBuffer[] minValues = minClustering != null ? minClustering.getBufferArray() : EMPTY_CLUSTERING;
|
||||
ByteBuffer[] maxValues = maxClustering != null ? maxClustering.getBufferArray() : EMPTY_CLUSTERING;
|
||||
ByteBuffer[] minValues = minClustering != null ? minClustering.retainable().getBufferArray() : EMPTY_CLUSTERING;
|
||||
ByteBuffer[] maxValues = maxClustering != null ? maxClustering.retainable().getBufferArray() : EMPTY_CLUSTERING;
|
||||
Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class);
|
||||
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
|
||||
components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.metrics;
|
||||
|
||||
import com.codahale.metrics.Gauge;
|
||||
|
||||
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
|
||||
|
||||
public class MinMaxAvgMetric
|
||||
{
|
||||
private final MetricNameFactory factory;
|
||||
private final String namePrefix;
|
||||
|
||||
final Gauge<Long> minGauge;
|
||||
final Gauge<Long> maxGauge;
|
||||
final Gauge<Double> avgGauge;
|
||||
final Gauge<Double> stddevGauge;
|
||||
final Gauge<Integer> numSamplesGauge;
|
||||
|
||||
private long min;
|
||||
private long max;
|
||||
private long sum;
|
||||
private long sumSquares;
|
||||
private int numSamples;
|
||||
|
||||
public MinMaxAvgMetric(MetricNameFactory factory, String namePrefix)
|
||||
{
|
||||
this.factory = factory;
|
||||
this.namePrefix = namePrefix;
|
||||
|
||||
minGauge = Metrics.register(factory.createMetricName(namePrefix + "Min"), () -> min);
|
||||
maxGauge = Metrics.register(factory.createMetricName(namePrefix + "Max"), () -> max);
|
||||
avgGauge = Metrics.register(factory.createMetricName(namePrefix + "Avg"), () -> numSamples > 0 ? ((double) sum) / numSamples : 0);
|
||||
stddevGauge = Metrics.register(factory.createMetricName(namePrefix + "StdDev"), () -> stddev());
|
||||
numSamplesGauge = Metrics.register(factory.createMetricName(namePrefix + "NumSamples"), () -> numSamples);
|
||||
}
|
||||
|
||||
public void release()
|
||||
{
|
||||
Metrics.remove(factory.createMetricName(namePrefix + "Min"));
|
||||
Metrics.remove(factory.createMetricName(namePrefix + "Max"));
|
||||
Metrics.remove(factory.createMetricName(namePrefix + "Avg"));
|
||||
Metrics.remove(factory.createMetricName(namePrefix + "StdDev"));
|
||||
Metrics.remove(factory.createMetricName(namePrefix + "NumSamples"));
|
||||
}
|
||||
|
||||
public void reset()
|
||||
{
|
||||
sum = 0;
|
||||
sumSquares = 0;
|
||||
max = Long.MIN_VALUE;
|
||||
min = Long.MAX_VALUE;
|
||||
numSamples = 0;
|
||||
}
|
||||
|
||||
public void update(long value)
|
||||
{
|
||||
max = max > value ? max : value;
|
||||
min = min < value ? min : value;
|
||||
sum += value;
|
||||
sumSquares += value * value;
|
||||
numSamples++;
|
||||
}
|
||||
|
||||
private Double stddev()
|
||||
{
|
||||
if (numSamples > 0)
|
||||
{
|
||||
double avgSquare = ((double) sumSquares) / numSamples;
|
||||
double avg = ((double) sum) / numSamples;
|
||||
return Math.sqrt(avgSquare - avg * avg);
|
||||
}
|
||||
else
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return "{" +
|
||||
"min=" + min +
|
||||
", max=" + max +
|
||||
", avg=" + (sum * 1.0 / numSamples) +
|
||||
", stdDev=" + stddev() +
|
||||
", numSamples=" + numSamples +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.metrics;
|
||||
|
||||
import com.codahale.metrics.Counter;
|
||||
|
||||
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
|
||||
|
||||
public class TrieMemtableMetricsView
|
||||
{
|
||||
private static final String UNCONTENDED_PUTS = "Uncontended memtable puts";
|
||||
private static final String CONTENDED_PUTS = "Contended memtable puts";
|
||||
private static final String CONTENTION_TIME = "Contention time";
|
||||
private static final String LAST_FLUSH_SHARD_SIZES = "Shard sizes during last flush";
|
||||
|
||||
// the number of memtable puts that did not need to wait on write lock
|
||||
public final Counter uncontendedPuts;
|
||||
|
||||
// the number of memtable puts that needed to wait on write lock
|
||||
public final Counter contendedPuts;
|
||||
|
||||
// shard put contention measurements
|
||||
public final LatencyMetrics contentionTime;
|
||||
|
||||
// shard sizes distribution
|
||||
public final MinMaxAvgMetric lastFlushShardDataSizes;
|
||||
|
||||
private final TrieMemtableMetricNameFactory factory;
|
||||
|
||||
public TrieMemtableMetricsView(String keyspace, String table)
|
||||
{
|
||||
factory = new TrieMemtableMetricNameFactory(keyspace, table);
|
||||
|
||||
uncontendedPuts = Metrics.counter(factory.createMetricName(UNCONTENDED_PUTS));
|
||||
contendedPuts = Metrics.counter(factory.createMetricName(CONTENDED_PUTS));
|
||||
contentionTime = new LatencyMetrics(factory, CONTENTION_TIME);
|
||||
lastFlushShardDataSizes = new MinMaxAvgMetric(factory, LAST_FLUSH_SHARD_SIZES);
|
||||
}
|
||||
|
||||
public void release()
|
||||
{
|
||||
Metrics.remove(factory.createMetricName(UNCONTENDED_PUTS));
|
||||
Metrics.remove(factory.createMetricName(CONTENDED_PUTS));
|
||||
contentionTime.release();
|
||||
lastFlushShardDataSizes.release();
|
||||
}
|
||||
|
||||
static class TrieMemtableMetricNameFactory implements MetricNameFactory
|
||||
{
|
||||
private final String keyspace;
|
||||
private final String table;
|
||||
|
||||
TrieMemtableMetricNameFactory(String keyspace, String table)
|
||||
{
|
||||
this.keyspace = keyspace;
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
public CassandraMetricsRegistry.MetricName createMetricName(String metricName)
|
||||
{
|
||||
String groupName = TableMetrics.class.getPackage().getName();
|
||||
String type = "TrieMemtable";
|
||||
|
||||
StringBuilder mbeanName = new StringBuilder();
|
||||
mbeanName.append(groupName).append(":");
|
||||
mbeanName.append("type=").append(type);
|
||||
mbeanName.append(",keyspace=").append(keyspace);
|
||||
mbeanName.append(",scope=").append(table);
|
||||
mbeanName.append(",name=").append(metricName);
|
||||
|
||||
return new CassandraMetricsRegistry.MetricName(groupName, type, metricName, keyspace + "." + table, mbeanName.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -681,13 +681,14 @@ public class ByteBufferUtil
|
|||
|
||||
public static boolean canMinimize(ByteBuffer buf)
|
||||
{
|
||||
return buf != null && (buf.capacity() > buf.remaining() || !buf.hasArray());
|
||||
return buf != null && (!buf.hasArray() || buf.array().length > buf.remaining());
|
||||
// Note: buf.array().length is different from buf.capacity() for sliced buffers.
|
||||
}
|
||||
|
||||
/** trims size of bytebuffer to exactly number of bytes in it, to do not hold too much memory */
|
||||
public static ByteBuffer minimalBufferFor(ByteBuffer buf)
|
||||
{
|
||||
return buf.capacity() > buf.remaining() || !buf.hasArray() ? ByteBuffer.wrap(getArray(buf)) : buf;
|
||||
return !buf.hasArray() || buf.array().length > buf.remaining() ? ByteBuffer.wrap(getArray(buf)) : buf;
|
||||
}
|
||||
|
||||
public static ByteBuffer[] minimizeBuffers(ByteBuffer[] src)
|
||||
|
|
|
|||
|
|
@ -668,6 +668,7 @@ Examples:
|
|||
|-8.1e-2000 | 999 | -0.0810|. -09 90 | 42·03 E7·77 DA·00
|
||||
|8.1e-2000 | -999 | 0.0810 |. 08 10 | BE·FC 19·88 8A·00
|
||||
|8.1e2000 | 1001 | 0.0810 |. 08 10 | C2·03 E9·88 8A·00
|
||||
|
||||
(mexp stands for “modulated exponent”, i.e. exponent * sign)
|
||||
|
||||
The values are prefix-free, because no exponent’s encoding can be a prefix of another, and the mantissas can never have
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ import org.apache.cassandra.utils.SearchIterator;
|
|||
|
||||
public abstract class EnsureOnHeap extends Transformation
|
||||
{
|
||||
public static final EnsureOnHeap NOOP = new NoOp();
|
||||
|
||||
public abstract DecoratedKey applyToPartitionKey(DecoratedKey key);
|
||||
public abstract UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition);
|
||||
public abstract SearchIterator<Clustering<?>, Row> applyToPartition(SearchIterator<Clustering<?>, Row> partition);
|
||||
|
|
|
|||
|
|
@ -30,8 +30,6 @@ import static org.apache.cassandra.utils.Shared.Scope.SIMULATION;
|
|||
|
||||
public class HeapPool extends MemtablePool
|
||||
{
|
||||
private static final EnsureOnHeap ENSURE_NOOP = new EnsureOnHeap.NoOp();
|
||||
|
||||
public HeapPool(long maxOnHeapMemory, float cleanupThreshold, MemtableCleaner cleaner)
|
||||
{
|
||||
super(maxOnHeapMemory, 0, cleanupThreshold, cleaner);
|
||||
|
|
@ -59,7 +57,7 @@ public class HeapPool extends MemtablePool
|
|||
|
||||
public EnsureOnHeap ensureOnHeap()
|
||||
{
|
||||
return ENSURE_NOOP;
|
||||
return EnsureOnHeap.NOOP;
|
||||
}
|
||||
|
||||
public Cloner cloner(OpOrder.Group opGroup)
|
||||
|
|
@ -124,7 +122,7 @@ public class HeapPool extends MemtablePool
|
|||
@Override
|
||||
public EnsureOnHeap ensureOnHeap()
|
||||
{
|
||||
return ENSURE_NOOP;
|
||||
return EnsureOnHeap.NOOP;
|
||||
}
|
||||
|
||||
public Cloner cloner(OpOrder.Group opGroup)
|
||||
|
|
|
|||
|
|
@ -68,13 +68,15 @@ row_index_read_size_fail_threshold: 8192KiB
|
|||
memtable:
|
||||
configurations:
|
||||
skiplist:
|
||||
inherits: default
|
||||
class_name: SkipListMemtable
|
||||
trie:
|
||||
class_name: TrieMemtable
|
||||
parameters:
|
||||
shards: 4
|
||||
skiplist_sharded:
|
||||
class_name: ShardedSkipListMemtable
|
||||
parameters:
|
||||
serialize_writes: false
|
||||
shards: 4
|
||||
skiplist_sharded_locking:
|
||||
inherits: skiplist_sharded
|
||||
parameters:
|
||||
|
|
@ -82,7 +84,6 @@ memtable:
|
|||
skiplist_remapped:
|
||||
inherits: skiplist
|
||||
test_fullname:
|
||||
inherits: default
|
||||
class_name: org.apache.cassandra.db.memtable.TestMemtable
|
||||
test_shortname:
|
||||
class_name: TestMemtable
|
||||
|
|
@ -106,3 +107,7 @@ memtable:
|
|||
class_name: org.apache.cassandra.cql3.validation.operations.CreateTest$InvalidMemtableFactoryMethod
|
||||
test_invalid_factory_field:
|
||||
class_name: org.apache.cassandra.cql3.validation.operations.CreateTest$InvalidMemtableFactoryField
|
||||
test_memtable_metrics:
|
||||
class_name: TrieMemtable
|
||||
# Note: keep the memtable configuration at the end of the file, so that the default mapping can be changed without
|
||||
# duplicating the whole section above.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Note: this attaches at the end of cassandra.yaml, where the memtable configuration setting must be.
|
||||
default:
|
||||
inherits: trie
|
||||
|
|
@ -50,8 +50,8 @@ import org.apache.cassandra.db.marshal.BytesType;
|
|||
import org.apache.cassandra.db.marshal.CompositeType;
|
||||
import org.apache.cassandra.db.marshal.Int32Type;
|
||||
import org.apache.cassandra.db.marshal.MapType;
|
||||
import org.apache.cassandra.db.partitions.AbstractBTreePartition;
|
||||
import org.apache.cassandra.db.partitions.AtomicBTreePartition;
|
||||
import org.apache.cassandra.db.partitions.BTreePartitionData;
|
||||
import org.apache.cassandra.db.partitions.PartitionUpdate;
|
||||
import org.apache.cassandra.db.rows.BTreeRow;
|
||||
import org.apache.cassandra.db.rows.BufferCell;
|
||||
|
|
@ -240,7 +240,7 @@ public class AtomicBTreePartitionUpdateBench
|
|||
try (BulkIterator<Row> iter = BulkIterator.of(insertBuffer))
|
||||
{
|
||||
Object[] tree = BTree.build(iter, rowCount, UpdateFunction.noOp());
|
||||
return PartitionUpdate.unsafeConstruct(metadata, decoratedKey, AbstractBTreePartition.unsafeConstructHolder(partitionColumns, tree, DeletionInfo.LIVE, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS), NO_DELETION_INFO, false);
|
||||
return PartitionUpdate.unsafeConstruct(metadata, decoratedKey, BTreePartitionData.unsafeConstruct(partitionColumns, tree, DeletionInfo.LIVE, Rows.EMPTY_STATIC_ROW, EncodingStats.NO_STATS), NO_DELETION_INFO, false);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -342,12 +342,7 @@ public class AtomicBTreePartitionUpdateBench
|
|||
public ByteBuffer allocate(int size)
|
||||
{
|
||||
if (invalidateOn > 0 && --invalidateOn == 0)
|
||||
{
|
||||
AbstractBTreePartition.Holder holder = update.unsafeGetHolder();
|
||||
if (!BTree.isEmpty(holder.tree))
|
||||
update.unsafeSetHolder(AbstractBTreePartition.unsafeConstructHolder(
|
||||
holder.columns, Arrays.copyOf(holder.tree, holder.tree.length), holder.deletionInfo, holder.staticRow, holder.stats));
|
||||
}
|
||||
BTreePartitionData.unsafeInvalidate(update);
|
||||
return ByteBuffer.allocate(size);
|
||||
}
|
||||
};
|
||||
|
|
@ -397,7 +392,7 @@ public class AtomicBTreePartitionUpdateBench
|
|||
ThreadLocalRandom.current().nextLong();
|
||||
}
|
||||
invokeBefore.accept(this);
|
||||
update.addAllWithSizeDelta(insert[index], cloner, NO_ORDER.getCurrent(), UpdateTransaction.NO_OP);
|
||||
update.addAll(insert[index], cloner, NO_ORDER.getCurrent(), UpdateTransaction.NO_OP);
|
||||
return true;
|
||||
}
|
||||
finally
|
||||
|
|
@ -405,7 +400,7 @@ public class AtomicBTreePartitionUpdateBench
|
|||
if (state.addAndGet(0x100000L) == ((((long)ifGeneration) << 40) | (((long)insert.length) << 20) | insert.length))
|
||||
{
|
||||
activeThreads.set(0);
|
||||
update.unsafeSetHolder(AbstractBTreePartition.unsafeGetEmptyHolder());
|
||||
update.unsafeSetHolder(BTreePartitionData.unsafeGetEmpty());
|
||||
// reset the state and rollover the generation
|
||||
state.set((ifGeneration + 1L) << 40);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,11 +80,21 @@ public abstract class ReadTest extends SimpleTableWriter
|
|||
// don't flush
|
||||
}
|
||||
|
||||
System.err.format("%s sstables, total %s, %,d partitions. Mean write latency %.2f ms\n",
|
||||
cfs.getLiveSSTables().size(),
|
||||
FBUtilities.prettyPrintMemory(cfs.metric.liveDiskSpaceUsed.getCount()),
|
||||
cfs.metric.estimatedPartitionCount.getValue(),
|
||||
cfs.metric.writeLatency.latency.getSnapshot().getMean());
|
||||
// Needed to stabilize sstable count for off-cache sized tests (e.g. count = 100_000_000)
|
||||
while (cfs.getLiveSSTables().size() >= 15)
|
||||
{
|
||||
cfs.enableAutoCompaction(true);
|
||||
cfs.disableAutoCompaction();
|
||||
System.err.format("%s sstables, total %s, %,d partitions. Mean write latency %.2f ms\n",
|
||||
cfs.getLiveSSTables().size(),
|
||||
FBUtilities.prettyPrintMemory(cfs.metric.liveDiskSpaceUsed.getCount()),
|
||||
cfs.metric.estimatedPartitionCount.getValue(),
|
||||
cfs.metric.writeLatency.latency.getSnapshot().getMean());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ public class MemtableTrieWriteBench
|
|||
@Param({"8"})
|
||||
int keyLength = 8;
|
||||
|
||||
final static MemtableTrie.UpsertTransformer<Byte, Byte> resolver = (x, y) -> x;
|
||||
final static MemtableTrie.UpsertTransformer<Byte, Byte> resolver = (x, y) -> y;
|
||||
|
||||
// Set this to true to print the trie sizes after insertions for sanity checking.
|
||||
// This might affect the timings, do not commit with this set to true.
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import java.util.UUID;
|
|||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import org.apache.cassandra.Util;
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.cql3.CQLTester;
|
||||
import org.apache.cassandra.db.ColumnFamilyStore;
|
||||
|
|
@ -29,7 +30,6 @@ import org.apache.cassandra.db.Keyspace;
|
|||
import org.apache.cassandra.db.memtable.Memtable;
|
||||
import org.apache.cassandra.db.memtable.SkipListMemtable;
|
||||
import org.apache.cassandra.db.memtable.TestMemtable;
|
||||
import org.apache.cassandra.dht.OrderPreservingPartitioner;
|
||||
import org.apache.cassandra.exceptions.ConfigurationException;
|
||||
import org.apache.cassandra.exceptions.InvalidRequestException;
|
||||
import org.apache.cassandra.exceptions.SyntaxException;
|
||||
|
|
@ -268,9 +268,9 @@ public class AlterTest extends CQLTester
|
|||
InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
|
||||
InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.4");
|
||||
metadata.updateHostId(UUID.randomUUID(), local);
|
||||
metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("A"), local);
|
||||
metadata.updateNormalToken(Util.token("A"), local);
|
||||
metadata.updateHostId(UUID.randomUUID(), remote);
|
||||
metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("B"), remote);
|
||||
metadata.updateNormalToken(Util.token("B"), remote);
|
||||
|
||||
// With two datacenters we should respect anything passed in as a manual override
|
||||
String ks1 = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'NetworkTopologyStrategy', 'replication_factor' : 1, '" + DATA_CENTER_REMOTE + "': 3}");
|
||||
|
|
@ -326,9 +326,9 @@ public class AlterTest extends CQLTester
|
|||
InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
|
||||
InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.4");
|
||||
metadata.updateHostId(UUID.randomUUID(), local);
|
||||
metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("A"), local);
|
||||
metadata.updateNormalToken(Util.token("A"), local);
|
||||
metadata.updateHostId(UUID.randomUUID(), remote);
|
||||
metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("B"), remote);
|
||||
metadata.updateNormalToken(Util.token("B"), remote);
|
||||
|
||||
// Let's create a keyspace first with SimpleStrategy
|
||||
String ks1 = createKeyspace("CREATE KEYSPACE %s WITH replication={ 'class' : 'SimpleStrategy', 'replication_factor' : 2}");
|
||||
|
|
@ -363,9 +363,9 @@ public class AlterTest extends CQLTester
|
|||
InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
|
||||
InetAddressAndPort remote = InetAddressAndPort.getByName("127.0.0.4");
|
||||
metadata.updateHostId(UUID.randomUUID(), local);
|
||||
metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("A"), local);
|
||||
metadata.updateNormalToken(Util.token("A"), local);
|
||||
metadata.updateHostId(UUID.randomUUID(), remote);
|
||||
metadata.updateNormalToken(new OrderPreservingPartitioner.StringToken("B"), remote);
|
||||
metadata.updateNormalToken(Util.token("B"), remote);
|
||||
|
||||
DatabaseDescriptor.setDefaultKeyspaceRF(3);
|
||||
|
||||
|
|
@ -560,6 +560,7 @@ public class AlterTest extends CQLTester
|
|||
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))");
|
||||
assertSame(MemtableParams.DEFAULT.factory(), getCurrentColumnFamilyStore().metadata().params.memtable.factory());
|
||||
assertSchemaOption("memtable", null);
|
||||
Class<? extends Memtable> defaultClass = getCurrentColumnFamilyStore().getTracker().getView().getCurrentMemtable().getClass();
|
||||
|
||||
testMemtableConfig("skiplist", SkipListMemtable.FACTORY, SkipListMemtable.class);
|
||||
testMemtableConfig("test_fullname", TestMemtable.FACTORY, SkipListMemtable.class);
|
||||
|
|
@ -570,7 +571,7 @@ public class AlterTest extends CQLTester
|
|||
+ " WITH compression = {'class': 'LZ4Compressor'};");
|
||||
assertSchemaOption("memtable", "test_shortname");
|
||||
|
||||
testMemtableConfig("default", MemtableParams.DEFAULT.factory(), SkipListMemtable.class);
|
||||
testMemtableConfig("default", MemtableParams.DEFAULT.factory(), defaultClass);
|
||||
|
||||
|
||||
assertAlterTableThrowsException(ConfigurationException.class,
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import org.apache.cassandra.db.Mutation;
|
|||
import org.apache.cassandra.db.memtable.Memtable;
|
||||
import org.apache.cassandra.db.memtable.SkipListMemtable;
|
||||
import org.apache.cassandra.db.memtable.TestMemtable;
|
||||
import org.apache.cassandra.db.memtable.TrieMemtable;
|
||||
import org.apache.cassandra.db.partitions.Partition;
|
||||
import org.apache.cassandra.exceptions.ConfigurationException;
|
||||
import org.apache.cassandra.exceptions.InvalidRequestException;
|
||||
|
|
@ -596,14 +597,16 @@ public class CreateTest extends CQLTester
|
|||
{
|
||||
createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))");
|
||||
assertSame(MemtableParams.DEFAULT.factory(), getCurrentColumnFamilyStore().metadata().params.memtable.factory());
|
||||
Class<? extends Memtable> defaultClass = getCurrentColumnFamilyStore().getTracker().getView().getCurrentMemtable().getClass();
|
||||
|
||||
assertSchemaOption("memtable", null);
|
||||
|
||||
testMemtableConfig("skiplist", SkipListMemtable.FACTORY, SkipListMemtable.class);
|
||||
testMemtableConfig("trie", MemtableParams.get("trie").factory(), TrieMemtable.class);
|
||||
testMemtableConfig("skiplist_remapped", SkipListMemtable.FACTORY, SkipListMemtable.class);
|
||||
testMemtableConfig("test_fullname", TestMemtable.FACTORY, SkipListMemtable.class);
|
||||
testMemtableConfig("test_shortname", SkipListMemtable.FACTORY, SkipListMemtable.class);
|
||||
testMemtableConfig("default", MemtableParams.DEFAULT.factory(), SkipListMemtable.class);
|
||||
testMemtableConfig("default", MemtableParams.DEFAULT.factory(), defaultClass);
|
||||
|
||||
assertThrowsConfigurationException("The 'class_name' option must be specified.",
|
||||
"CREATE TABLE %s (a text, b int, c int, primary key (a, b))"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,234 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.db;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import org.apache.cassandra.db.marshal.ByteArrayAccessor;
|
||||
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
|
||||
import org.apache.cassandra.db.marshal.ValueAccessor;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
import org.apache.cassandra.utils.concurrent.ImmediateFuture;
|
||||
import org.apache.cassandra.utils.memory.MemtablePool;
|
||||
import org.apache.cassandra.utils.memory.NativeAllocator;
|
||||
import org.apache.cassandra.utils.memory.NativePool;
|
||||
import org.apache.cassandra.utils.memory.SlabAllocator;
|
||||
import org.apache.cassandra.utils.memory.SlabPool;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class ClusteringPrefixTest
|
||||
{
|
||||
@Test
|
||||
public void arrayTopAndBottom()
|
||||
{
|
||||
Assert.assertTrue(ArrayClusteringBound.BOTTOM.isBottom());
|
||||
Assert.assertFalse(ArrayClusteringBound.BOTTOM.isTop());
|
||||
Assert.assertTrue(ArrayClusteringBound.TOP.isTop());
|
||||
Assert.assertFalse(ArrayClusteringBound.TOP.isBottom());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void bufferTopAndBottom()
|
||||
{
|
||||
Assert.assertTrue(BufferClusteringBound.BOTTOM.isBottom());
|
||||
Assert.assertFalse(BufferClusteringBound.BOTTOM.isTop());
|
||||
Assert.assertTrue(BufferClusteringBound.TOP.isTop());
|
||||
Assert.assertFalse(BufferClusteringBound.TOP.isBottom());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRetainableArray()
|
||||
{
|
||||
testRetainable(ByteArrayAccessor.instance.factory(), x -> new byte[][] {x.getBytes(StandardCharsets.UTF_8)});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRetainableOnHeap()
|
||||
{
|
||||
testRetainable(ByteBufferAccessor.instance.factory(), x -> new ByteBuffer[] {ByteBufferUtil.bytes(x)});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRetainableOnHeapSliced()
|
||||
{
|
||||
for (int prepend = 0; prepend < 3; ++prepend)
|
||||
{
|
||||
for (int append = 0; append < 3; ++append)
|
||||
{
|
||||
testRetainable(ByteBufferAccessor.instance.factory(),
|
||||
slicingAllocator(prepend, append));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Function<String, ByteBuffer[]> slicingAllocator(int prepend, int append)
|
||||
{
|
||||
return x ->
|
||||
{
|
||||
ByteBuffer bytes = ByteBufferUtil.bytes(x);
|
||||
ByteBuffer sliced = ByteBuffer.allocate(bytes.remaining() + prepend + append);
|
||||
for (int i = 0; i < prepend; ++i)
|
||||
sliced.put((byte) ThreadLocalRandom.current().nextInt());
|
||||
sliced.put(bytes);
|
||||
bytes.flip();
|
||||
for (int i = 0; i < append; ++i)
|
||||
sliced.put((byte) ThreadLocalRandom.current().nextInt());
|
||||
sliced.position(prepend).limit(prepend + bytes.remaining());
|
||||
return new ByteBuffer[]{ sliced.slice() };
|
||||
};
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRetainableOffHeap()
|
||||
{
|
||||
testRetainable(ByteBufferAccessor.instance.factory(), x ->
|
||||
{
|
||||
ByteBuffer h = ByteBufferUtil.bytes(x);
|
||||
ByteBuffer v = ByteBuffer.allocateDirect(h.remaining());
|
||||
v.put(h);
|
||||
v.flip();
|
||||
return new ByteBuffer[] {v};
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRetainableOnHeapSlab() throws InterruptedException, TimeoutException
|
||||
{
|
||||
testRetainableSlab(true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRetainableOffHeapSlab() throws InterruptedException, TimeoutException
|
||||
{
|
||||
testRetainableSlab(false);
|
||||
}
|
||||
|
||||
public void testRetainableSlab(boolean onHeap) throws InterruptedException, TimeoutException
|
||||
{
|
||||
MemtablePool pool = new SlabPool(1L << 24, onHeap ? 0 : 1L << 24, 1.0f, () -> ImmediateFuture.success(false));
|
||||
SlabAllocator allocator = ((SlabAllocator) pool.newAllocator("test"));
|
||||
assert !allocator.allocate(1).isDirect() == onHeap;
|
||||
try
|
||||
{
|
||||
testRetainable(ByteBufferAccessor.instance.factory(), x ->
|
||||
{
|
||||
ByteBuffer h = ByteBufferUtil.bytes(x);
|
||||
ByteBuffer v = allocator.allocate(h.remaining());
|
||||
v.put(h);
|
||||
v.flip();
|
||||
return new ByteBuffer[] {v};
|
||||
});
|
||||
}
|
||||
finally
|
||||
{
|
||||
pool.shutdownAndWait(10, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRetainableNative() throws InterruptedException, TimeoutException
|
||||
{
|
||||
MemtablePool pool = new NativePool(1L << 24,1L << 24, 1.0f, () -> ImmediateFuture.success(false));
|
||||
NativeAllocator allocator = (NativeAllocator) pool.newAllocator("test");
|
||||
try
|
||||
{
|
||||
testRetainable(ByteBufferAccessor.instance.factory(),
|
||||
x -> new ByteBuffer[] {ByteBufferUtil.bytes(x)},
|
||||
x -> x.kind() == ClusteringPrefix.Kind.CLUSTERING
|
||||
? new NativeClustering(allocator, null, (Clustering<?>) x)
|
||||
: x);
|
||||
}
|
||||
finally
|
||||
{
|
||||
pool.shutdownAndWait(10, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
public <V> void testRetainable(ValueAccessor.ObjectFactory<V> factory,
|
||||
Function<String, V[]> allocator)
|
||||
{
|
||||
testRetainable(factory, allocator, null);
|
||||
}
|
||||
|
||||
public <V> void testRetainable(ValueAccessor.ObjectFactory<V> factory,
|
||||
Function<String, V[]> allocator,
|
||||
Function<ClusteringPrefix<V>, ClusteringPrefix<V>> mapper)
|
||||
{
|
||||
ClusteringPrefix<V>[] clusterings = new ClusteringPrefix[]
|
||||
{
|
||||
factory.clustering(),
|
||||
factory.staticClustering(),
|
||||
factory.clustering(allocator.apply("test")),
|
||||
factory.bound(ClusteringPrefix.Kind.INCL_START_BOUND, allocator.apply("testA")),
|
||||
factory.bound(ClusteringPrefix.Kind.INCL_END_BOUND, allocator.apply("testB")),
|
||||
factory.bound(ClusteringPrefix.Kind.EXCL_START_BOUND, allocator.apply("testC")),
|
||||
factory.bound(ClusteringPrefix.Kind.EXCL_END_BOUND, allocator.apply("testD")),
|
||||
factory.boundary(ClusteringPrefix.Kind.EXCL_END_INCL_START_BOUNDARY, allocator.apply("testE")),
|
||||
factory.boundary(ClusteringPrefix.Kind.INCL_END_EXCL_START_BOUNDARY, allocator.apply("testF")),
|
||||
};
|
||||
|
||||
if (mapper != null)
|
||||
clusterings = Arrays.stream(clusterings)
|
||||
.map(mapper)
|
||||
.toArray(ClusteringPrefix[]::new);
|
||||
|
||||
testRetainable(clusterings);
|
||||
}
|
||||
|
||||
public void testRetainable(ClusteringPrefix<?>[] clusterings)
|
||||
{
|
||||
for (ClusteringPrefix<?> clustering : clusterings)
|
||||
{
|
||||
ClusteringPrefix<?> retainable = clustering.retainable();
|
||||
assertEquals(clustering, retainable);
|
||||
assertClusteringIsRetainable(retainable);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void assertClusteringIsRetainable(ClusteringPrefix<?> clustering)
|
||||
{
|
||||
if (clustering instanceof AbstractArrayClusteringPrefix)
|
||||
return; // has to be on-heap and minimized
|
||||
|
||||
assertTrue(clustering instanceof AbstractBufferClusteringPrefix);
|
||||
AbstractBufferClusteringPrefix abcf = (AbstractBufferClusteringPrefix) clustering;
|
||||
ByteBuffer[] buffers = abcf.getBufferArray();
|
||||
for (ByteBuffer b : buffers)
|
||||
{
|
||||
assertFalse(b.isDirect());
|
||||
assertTrue(b.hasArray());
|
||||
assertEquals(b.capacity(), b.remaining());
|
||||
assertEquals(0, b.arrayOffset());
|
||||
assertEquals(b.capacity(), b.array().length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -57,7 +57,8 @@ public class MemtableQuickTest extends CQLTester
|
|||
{
|
||||
return ImmutableList.of("skiplist",
|
||||
"skiplist_sharded",
|
||||
"skiplist_sharded_locking");
|
||||
"skiplist_sharded_locking",
|
||||
"trie");
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
|
|
|
|||
|
|
@ -20,14 +20,11 @@ package org.apache.cassandra.db.memtable;
|
|||
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import org.apache.cassandra.config.Config;
|
||||
import org.apache.cassandra.utils.memory.MemtablePool;
|
||||
import org.apache.cassandra.utils.memory.SlabPool;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class MemtableSizeHeapBuffersTest extends MemtableSizeTestBase
|
||||
{
|
||||
// Overrides CQLTester.setUpClass to run before it
|
||||
|
|
@ -43,6 +40,6 @@ public class MemtableSizeHeapBuffersTest extends MemtableSizeTestBase
|
|||
MemtablePool memoryPool = AbstractAllocatorMemtable.MEMORY_POOL;
|
||||
logger.info("Memtable pool {} off-heap limit {}", memoryPool, memoryPool.offHeap.limit);
|
||||
Assert.assertTrue(memoryPool instanceof SlabPool);
|
||||
Assert.assertTrue(memoryPool.offHeap.limit == 0);
|
||||
Assert.assertEquals(0, memoryPool.offHeap.limit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,14 +20,11 @@ package org.apache.cassandra.db.memtable;
|
|||
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import org.apache.cassandra.config.Config;
|
||||
import org.apache.cassandra.utils.memory.MemtablePool;
|
||||
import org.apache.cassandra.utils.memory.SlabPool;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class MemtableSizeOffheapBuffersTest extends MemtableSizeTestBase
|
||||
{
|
||||
// Overrides CQLTester.setUpClass to run before it
|
||||
|
|
@ -46,4 +43,4 @@ public class MemtableSizeOffheapBuffersTest extends MemtableSizeTestBase
|
|||
Assert.assertTrue(memoryPool instanceof SlabPool);
|
||||
Assert.assertTrue(memoryPool.offHeap.limit > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,14 +20,11 @@ package org.apache.cassandra.db.memtable;
|
|||
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import org.apache.cassandra.config.Config;
|
||||
import org.apache.cassandra.utils.memory.MemtablePool;
|
||||
import org.apache.cassandra.utils.memory.NativePool;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class MemtableSizeOffheapObjectsTest extends MemtableSizeTestBase
|
||||
{
|
||||
// Overrides CQLTester.setUpClass to run before it
|
||||
|
|
@ -41,7 +38,7 @@ public class MemtableSizeOffheapObjectsTest extends MemtableSizeTestBase
|
|||
void checkMemtablePool()
|
||||
{
|
||||
MemtablePool memoryPool = AbstractAllocatorMemtable.MEMORY_POOL;
|
||||
System.out.println("Memtable pool " + memoryPool + " off-heap limit " + memoryPool.offHeap.limit);
|
||||
logger.info("Memtable pool {} off-heap limit {}", memoryPool, memoryPool.offHeap.limit);
|
||||
Assert.assertTrue(memoryPool instanceof NativePool);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,10 +21,10 @@ package org.apache.cassandra.db.memtable;
|
|||
import java.lang.reflect.Field;
|
||||
import java.util.List;
|
||||
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -41,6 +41,7 @@ import org.github.jamm.MemoryMeter;
|
|||
|
||||
// Note: This test can be run in idea with the allocation type configured in the test yaml and memtable using the
|
||||
// value memtableClass is initialized with.
|
||||
@RunWith(Parameterized.class)
|
||||
public class MemtableSizeTestBase extends CQLTester
|
||||
{
|
||||
// Note: To see a printout of the usage for each object, add .enableDebug() here (most useful with smaller number of
|
||||
|
|
@ -63,7 +64,8 @@ public class MemtableSizeTestBase extends CQLTester
|
|||
public static List<Object> parameters()
|
||||
{
|
||||
return ImmutableList.of("skiplist",
|
||||
"skiplist_sharded");
|
||||
"skiplist_sharded",
|
||||
"trie");
|
||||
}
|
||||
|
||||
// Must be within 3% of the real usage. We are actually more precise than this, but the threshold is set higher to
|
||||
|
|
@ -84,7 +86,7 @@ public class MemtableSizeTestBase extends CQLTester
|
|||
}
|
||||
catch (NoSuchFieldException | IllegalAccessException e)
|
||||
{
|
||||
throw Throwables.propagate(e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
CQLTester.setUpClass();
|
||||
|
|
@ -165,10 +167,15 @@ public class MemtableSizeTestBase extends CQLTester
|
|||
|
||||
long expectedHeap = deepSizeAfter - deepSizeBefore;
|
||||
long max_difference = MAX_DIFFERENCE_PERCENT * expectedHeap / 100;
|
||||
long trie_overhead = memtable instanceof TrieMemtable ? ((TrieMemtable) memtable).unusedReservedMemory() : 0;
|
||||
switch (DatabaseDescriptor.getMemtableAllocationType())
|
||||
{
|
||||
case heap_buffers:
|
||||
max_difference += SLAB_OVERHEAD;
|
||||
actualHeap += trie_overhead; // adjust trie memory with unused buffer space if on-heap
|
||||
break;
|
||||
case unslabbed_heap_buffers:
|
||||
actualHeap += trie_overhead; // adjust trie memory with unused buffer space if on-heap
|
||||
break;
|
||||
}
|
||||
String message = String.format("Expected heap usage close to %s, got %s, %s difference.\n",
|
||||
|
|
@ -196,4 +203,4 @@ public class MemtableSizeTestBase extends CQLTester
|
|||
execute(String.format("DROP KEYSPACE IF EXISTS %s", keyspace));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,14 +20,11 @@ package org.apache.cassandra.db.memtable;
|
|||
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import org.apache.cassandra.config.Config;
|
||||
import org.apache.cassandra.utils.memory.HeapPool;
|
||||
import org.apache.cassandra.utils.memory.MemtablePool;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class MemtableSizeUnslabbedTest extends MemtableSizeTestBase
|
||||
{
|
||||
// Overrides CQLTester.setUpClass to run before it
|
||||
|
|
@ -44,4 +41,4 @@ public class MemtableSizeUnslabbedTest extends MemtableSizeTestBase
|
|||
logger.info("Memtable pool {} off-heap limit {}", memoryPool, memoryPool.offHeap.limit);
|
||||
Assert.assertTrue(memoryPool instanceof HeapPool);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.db.memtable;
|
||||
|
||||
import java.io.IOException;
|
||||
import javax.management.Attribute;
|
||||
import javax.management.AttributeNotFoundException;
|
||||
import javax.management.InstanceNotFoundException;
|
||||
import javax.management.InvalidAttributeValueException;
|
||||
import javax.management.MBeanException;
|
||||
import javax.management.MalformedObjectNameException;
|
||||
import javax.management.ObjectName;
|
||||
import javax.management.ReflectionException;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import org.apache.cassandra.cql3.CQLTester;
|
||||
import org.apache.cassandra.utils.FBUtilities;
|
||||
|
||||
import static org.apache.cassandra.db.memtable.AbstractShardedMemtable.SHARDED_MEMTABLE_CONFIG_OBJECT_NAME;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class ShardedMemtableConfigTest extends CQLTester
|
||||
{
|
||||
@BeforeClass
|
||||
public static void setup() throws Exception
|
||||
{
|
||||
startJMXServer();
|
||||
createMBeanServerConnection();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDefaultShardCountSetByJMX() throws MalformedObjectNameException, ReflectionException, AttributeNotFoundException, InstanceNotFoundException, MBeanException, IOException, InvalidAttributeValueException, InterruptedException
|
||||
{
|
||||
// check the default, but also make sure the class is initialized if the default memtable is not sharded
|
||||
assertEquals(FBUtilities.getAvailableProcessors(), AbstractShardedMemtable.getDefaultShardCount());
|
||||
jmxConnection.setAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), new Attribute("DefaultShardCount", "7"));
|
||||
assertEquals(7, AbstractShardedMemtable.getDefaultShardCount());
|
||||
assertEquals("7", jmxConnection.getAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), "DefaultShardCount"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAutoShardCount() throws MalformedObjectNameException, ReflectionException, AttributeNotFoundException, InstanceNotFoundException, MBeanException, IOException, InvalidAttributeValueException
|
||||
{
|
||||
AbstractShardedMemtable.getDefaultShardCount(); // initialize class
|
||||
jmxConnection.setAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), new Attribute("DefaultShardCount", "auto"));
|
||||
assertEquals(FBUtilities.getAvailableProcessors(), AbstractShardedMemtable.getDefaultShardCount());
|
||||
assertEquals(Integer.toString(FBUtilities.getAvailableProcessors()),
|
||||
jmxConnection.getAttribute(new ObjectName(SHARDED_MEMTABLE_CONFIG_OBJECT_NAME), "DefaultShardCount"));
|
||||
}
|
||||
}
|
||||
|
|
@ -77,7 +77,7 @@ public class MemtableTriePutTest extends MemtableTrieTestBase
|
|||
Assert.assertNull(trie.get(ByteComparable.of(t2)));
|
||||
Assert.assertFalse(trie.reachedAllocatedSizeThreshold());
|
||||
|
||||
trie.advanceAllocatedPos(0x40001000); // over 1G
|
||||
trie.advanceAllocatedPos(MemtableTrie.ALLOCATED_SIZE_THRESHOLD + 0x1000);
|
||||
trie.putRecursive(ByteComparable.of(t2), t2, (x, y) -> y);
|
||||
Assert.assertEquals(t1, trie.get(ByteComparable.of(t1)));
|
||||
Assert.assertEquals(t2, trie.get(ByteComparable.of(t2)));
|
||||
|
|
@ -119,5 +119,7 @@ public class MemtableTriePutTest extends MemtableTrieTestBase
|
|||
Assert.assertEquals(t2, trie.get(ByteComparable.of(t2)));
|
||||
Assert.assertNull(trie.get(ByteComparable.of(t3)));
|
||||
Assert.assertTrue(trie.reachedAllocatedSizeThreshold());
|
||||
|
||||
trie.discardBuffers();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -313,6 +313,8 @@ public abstract class MemtableTrieTestBase
|
|||
|
||||
assertSameContent(trie, content);
|
||||
checkGet(trie, content);
|
||||
|
||||
trie.discardBuffers();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import java.util.function.Function;
|
|||
import org.agrona.DirectBuffer;
|
||||
|
||||
/**
|
||||
* Simple utility class for dumping the structure of a trie to string.
|
||||
* A class for dumping the structure of a trie to a graphviz/dot representation for making trie graphs.
|
||||
*/
|
||||
class TrieToDot<T> extends TriePathReconstructor implements Trie.Walker<T, String>
|
||||
{
|
||||
|
|
@ -17,6 +17,7 @@
|
|||
*/
|
||||
package org.apache.cassandra.io.sstable;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.CharacterCodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
|
@ -37,6 +38,7 @@ import org.apache.cassandra.schema.KeyspaceParams;
|
|||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class SSTableMetadataTest
|
||||
|
|
@ -218,9 +220,9 @@ public class SSTableMetadataTest
|
|||
{
|
||||
assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0)), "0col100");
|
||||
assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0)), "7col149");
|
||||
// make sure the clustering values are minimised
|
||||
assertTrue(sstable.getSSTableMetadata().minClusteringValues.get(0).capacity() < 50);
|
||||
assertTrue(sstable.getSSTableMetadata().maxClusteringValues.get(0).capacity() < 50);
|
||||
// make sure stats don't reference native or off-heap data
|
||||
assertBuffersAreRetainable(sstable.getSSTableMetadata().minClusteringValues);
|
||||
assertBuffersAreRetainable(sstable.getSSTableMetadata().maxClusteringValues);
|
||||
}
|
||||
String key = "row2";
|
||||
|
||||
|
|
@ -240,9 +242,39 @@ public class SSTableMetadataTest
|
|||
{
|
||||
assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0)), "0col100");
|
||||
assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0)), "9col298");
|
||||
// and make sure the clustering values are still minimised after compaction
|
||||
assertTrue(sstable.getSSTableMetadata().minClusteringValues.get(0).capacity() < 50);
|
||||
assertTrue(sstable.getSSTableMetadata().maxClusteringValues.get(0).capacity() < 50);
|
||||
// make sure stats don't reference native or off-heap data
|
||||
assertBuffersAreRetainable(sstable.getSSTableMetadata().minClusteringValues);
|
||||
assertBuffersAreRetainable(sstable.getSSTableMetadata().maxClusteringValues);
|
||||
}
|
||||
|
||||
key = "row3";
|
||||
new RowUpdateBuilder(store.metadata(), System.currentTimeMillis(), key)
|
||||
.addRangeTombstone("0", "7")
|
||||
.build()
|
||||
.apply();
|
||||
|
||||
store.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
|
||||
store.forceMajorCompaction();
|
||||
assertEquals(1, store.getLiveSSTables().size());
|
||||
for (SSTableReader sstable : store.getLiveSSTables())
|
||||
{
|
||||
assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0)), "0");
|
||||
assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0)), "9col298");
|
||||
// make sure stats don't reference native or off-heap data
|
||||
assertBuffersAreRetainable(sstable.getSSTableMetadata().minClusteringValues);
|
||||
assertBuffersAreRetainable(sstable.getSSTableMetadata().maxClusteringValues);
|
||||
}
|
||||
}
|
||||
|
||||
public static void assertBuffersAreRetainable(List<ByteBuffer> buffers)
|
||||
{
|
||||
for (ByteBuffer b : buffers)
|
||||
{
|
||||
assertFalse(b.isDirect());
|
||||
assertTrue(b.hasArray());
|
||||
assertEquals(b.capacity(), b.remaining());
|
||||
assertEquals(0, b.arrayOffset());
|
||||
assertEquals(b.capacity(), b.array().length);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,210 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.metrics;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.datastax.driver.core.Cluster;
|
||||
import com.datastax.driver.core.Session;
|
||||
import org.apache.cassandra.SchemaLoader;
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.config.OverrideConfigurationLoader;
|
||||
import org.apache.cassandra.db.ColumnFamilyStore;
|
||||
import org.apache.cassandra.db.memtable.AbstractShardedMemtable;
|
||||
import org.apache.cassandra.exceptions.ConfigurationException;
|
||||
import org.apache.cassandra.service.EmbeddedCassandraService;
|
||||
import org.apache.cassandra.service.StorageService;
|
||||
import org.jboss.byteman.contrib.bmunit.BMRule;
|
||||
import org.jboss.byteman.contrib.bmunit.BMRules;
|
||||
import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
|
||||
|
||||
import static org.hamcrest.Matchers.*;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
@RunWith(BMUnitRunner.class)
|
||||
public class TrieMemtableMetricsTest extends SchemaLoader
|
||||
{
|
||||
private static final int NUM_SHARDS = 13;
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(TrieMemtableMetricsTest.class);
|
||||
private static Session session;
|
||||
|
||||
private static final String KEYSPACE = "triememtable";
|
||||
private static final String TABLE = "metricstest";
|
||||
|
||||
@BeforeClass
|
||||
public static void loadSchema() throws ConfigurationException
|
||||
{
|
||||
// shadow superclass method; we'll call it directly
|
||||
// after tinkering with the Config
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void setup() throws ConfigurationException, IOException
|
||||
{
|
||||
OverrideConfigurationLoader.override((config) -> {
|
||||
config.partitioner = "Murmur3Partitioner";
|
||||
});
|
||||
System.setProperty(AbstractShardedMemtable.DEFAULT_SHARD_COUNT_PROPERTY, "" + NUM_SHARDS);
|
||||
|
||||
SchemaLoader.loadSchema();
|
||||
|
||||
EmbeddedCassandraService cassandra = new EmbeddedCassandraService();
|
||||
cassandra.start();
|
||||
|
||||
Cluster cluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(DatabaseDescriptor.getNativeTransportPort()).build();
|
||||
session = cluster.connect();
|
||||
|
||||
session.execute(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };", KEYSPACE));
|
||||
}
|
||||
|
||||
private ColumnFamilyStore recreateTable()
|
||||
{
|
||||
return recreateTable(TABLE);
|
||||
}
|
||||
|
||||
private ColumnFamilyStore recreateTable(String table)
|
||||
{
|
||||
session.execute(String.format("DROP TABLE IF EXISTS %s.%s", KEYSPACE, table));
|
||||
session.execute(String.format("CREATE TABLE IF NOT EXISTS %s.%s (id int, val1 text, val2 text, PRIMARY KEY(id, val1)) WITH MEMTABLE = 'test_memtable_metrics';", KEYSPACE, table));
|
||||
return ColumnFamilyStore.getIfExists(KEYSPACE, table);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRegularStatementsAreCounted()
|
||||
{
|
||||
ColumnFamilyStore cfs = recreateTable();
|
||||
TrieMemtableMetricsView metrics = getMemtableMetrics(cfs);
|
||||
assertEquals(0, metrics.contendedPuts.getCount());
|
||||
assertEquals(0, metrics.uncontendedPuts.getCount());
|
||||
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
session.execute(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TABLE, i, "val" + i, "val" + i));
|
||||
}
|
||||
|
||||
long allPuts = metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount();
|
||||
assertEquals(10, allPuts);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFlushRelatedMetrics() throws IOException, ExecutionException, InterruptedException
|
||||
{
|
||||
ColumnFamilyStore cfs = recreateTable();
|
||||
TrieMemtableMetricsView metrics = getMemtableMetrics(cfs);
|
||||
|
||||
StorageService.instance.forceKeyspaceFlush(KEYSPACE, TABLE);
|
||||
assertEquals(0, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
|
||||
|
||||
writeAndFlush(10);
|
||||
assertEquals(10, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
|
||||
|
||||
// verify that metrics survive flush / memtable switching
|
||||
writeAndFlush(10);
|
||||
assertEquals(20, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
|
||||
assertEquals(metrics.lastFlushShardDataSizes.toString(), NUM_SHARDS, metrics.lastFlushShardDataSizes.numSamplesGauge.getValue().intValue());
|
||||
}
|
||||
|
||||
@Test
|
||||
@BMRules(rules = { @BMRule(name = "Delay memtable update",
|
||||
targetClass = "MemtableTrie",
|
||||
targetMethod = "putSingleton",
|
||||
action = "java.lang.Thread.sleep(10)")})
|
||||
public void testContentionMetrics() throws IOException, ExecutionException, InterruptedException
|
||||
{
|
||||
ColumnFamilyStore cfs = recreateTable();
|
||||
TrieMemtableMetricsView metrics = getMemtableMetrics(cfs);
|
||||
assertEquals(0, (int) metrics.lastFlushShardDataSizes.numSamplesGauge.getValue());
|
||||
|
||||
StorageService.instance.forceKeyspaceFlush(KEYSPACE, TABLE);
|
||||
|
||||
writeAndFlush(100);
|
||||
|
||||
ByteArrayOutputStream stream = new ByteArrayOutputStream();
|
||||
metrics.contentionTime.latency.getSnapshot().dump(stream);
|
||||
|
||||
assertEquals(100, metrics.contendedPuts.getCount() + metrics.uncontendedPuts.getCount());
|
||||
assertThat(metrics.contendedPuts.getCount(), greaterThan(0L));
|
||||
assertThat(metrics.contentionTime.totalLatency.getCount(), greaterThan(0L));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMetricsCleanupOnDrop()
|
||||
{
|
||||
String tableName = TABLE + "_metrics_cleanup";
|
||||
CassandraMetricsRegistry registry = CassandraMetricsRegistry.Metrics;
|
||||
Supplier<Stream<String>> metrics = () -> registry.getNames().stream().filter(m -> m.contains(tableName));
|
||||
|
||||
// no metrics before creating
|
||||
assertEquals(0, metrics.get().count());
|
||||
|
||||
recreateTable(tableName);
|
||||
// some metrics
|
||||
assertTrue(metrics.get().count() > 0);
|
||||
|
||||
session.execute(String.format("DROP TABLE IF EXISTS %s.%s", KEYSPACE, tableName));
|
||||
// no metrics after drop
|
||||
assertEquals(metrics.get().collect(Collectors.joining(",")), 0, metrics.get().count());
|
||||
}
|
||||
|
||||
private TrieMemtableMetricsView getMemtableMetrics(ColumnFamilyStore cfs)
|
||||
{
|
||||
return new TrieMemtableMetricsView(cfs.keyspace.getName(), cfs.name);
|
||||
}
|
||||
|
||||
private void writeAndFlush(int rows) throws IOException, ExecutionException, InterruptedException
|
||||
{
|
||||
logger.info("writing {} rows", rows);
|
||||
Future[] futures = new Future[rows];
|
||||
for (int i = 0; i < rows; i++)
|
||||
{
|
||||
logger.info("writing {} row", i);
|
||||
futures[i] = session.executeAsync(String.format("INSERT INTO %s.%s (id, val1, val2) VALUES (%d, '%s', '%s')", KEYSPACE, TABLE, i, "val" + i, "val" + i));
|
||||
}
|
||||
for (int i = 0; i < rows; i++)
|
||||
{
|
||||
futures[i].get();
|
||||
logger.info("writing {} row completed", i);
|
||||
}
|
||||
logger.info("forcing flush");
|
||||
StorageService.instance.forceKeyspaceFlush(KEYSPACE, TABLE);
|
||||
logger.info("table flushed");
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void teardown()
|
||||
{
|
||||
session.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -39,7 +39,8 @@ public class BulkLoaderTest extends OfflineToolUtils
|
|||
|
||||
assertNoUnexpectedThreadsStarted(new String[] { "ObjectCleanerThread",
|
||||
"Shutdown-checker",
|
||||
"cluster[0-9]-connection-reaper-[0-9]" });
|
||||
"cluster[0-9]-connection-reaper-[0-9]" },
|
||||
false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -67,7 +68,8 @@ public class BulkLoaderTest extends OfflineToolUtils
|
|||
"Shutdown-checker",
|
||||
"cluster[0-9]-connection-reaper-[0-9]",
|
||||
"Attach Listener",
|
||||
"process reaper"});
|
||||
"process reaper"},
|
||||
false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -97,7 +99,8 @@ public class BulkLoaderTest extends OfflineToolUtils
|
|||
"Shutdown-checker",
|
||||
"cluster[0-9]-connection-reaper-[0-9]",
|
||||
"Attach Listener",
|
||||
"process reaper"});
|
||||
"process reaper"},
|
||||
false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -127,7 +130,8 @@ public class BulkLoaderTest extends OfflineToolUtils
|
|||
"Shutdown-checker",
|
||||
"cluster[0-9]-connection-reaper-[0-9]",
|
||||
"Attach Listener",
|
||||
"process reaper"});
|
||||
"process reaper"},
|
||||
false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ public class GetVersionTest extends OfflineToolUtils
|
|||
{
|
||||
ToolResult tool = ToolRunner.invokeClass(GetVersion.class);
|
||||
tool.assertOnCleanExit();
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
|
|||
|
|
@ -30,7 +30,11 @@ import java.util.Objects;
|
|||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
import com.google.common.collect.Iterables;
|
||||
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.io.util.File;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.BeforeClass;
|
||||
|
|
@ -74,7 +78,12 @@ public abstract class OfflineToolUtils
|
|||
"Attach Listener", // spawned in intellij IDEA
|
||||
};
|
||||
|
||||
public void assertNoUnexpectedThreadsStarted(String[] optionalThreadNames)
|
||||
static final String[] NON_DEFAULT_MEMTABLE_THREADS =
|
||||
{
|
||||
"((Native|Slab|Heap)Pool|Logged)Cleaner"
|
||||
};
|
||||
|
||||
public void assertNoUnexpectedThreadsStarted(String[] optionalThreadNames, boolean allowNonDefaultMemtableThreads)
|
||||
{
|
||||
ThreadMXBean threads = ManagementFactory.getThreadMXBean();
|
||||
|
||||
|
|
@ -87,10 +96,15 @@ public abstract class OfflineToolUtils
|
|||
.filter(Objects::nonNull)
|
||||
.map(ThreadInfo::getThreadName)
|
||||
.collect(Collectors.toSet());
|
||||
Iterable<String> optionalNames = optionalThreadNames != null
|
||||
? Arrays.asList(optionalThreadNames)
|
||||
: Collections.emptyList();
|
||||
if (allowNonDefaultMemtableThreads && DatabaseDescriptor.getMemtableConfigurations().containsKey("default"))
|
||||
optionalNames = Iterables.concat(optionalNames, Arrays.asList(NON_DEFAULT_MEMTABLE_THREADS));
|
||||
|
||||
List<Pattern> optional = optionalThreadNames != null
|
||||
? Arrays.stream(optionalThreadNames).map(Pattern::compile).collect(Collectors.toList())
|
||||
: Collections.emptyList();
|
||||
List<Pattern> optional = StreamSupport.stream(optionalNames.spliterator(), false)
|
||||
.map(Pattern::compile)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
current.removeAll(initial);
|
||||
|
||||
|
|
@ -219,7 +233,7 @@ public abstract class OfflineToolUtils
|
|||
|
||||
protected void assertCorrectEnvPostTest()
|
||||
{
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, true);
|
||||
assertSchemaLoaded();
|
||||
assertServerNotLoaded();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ public class SSTableExpiredBlockersTest extends OfflineToolUtils
|
|||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
assertEquals(1, tool.getExitCode());
|
||||
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
|
|||
|
|
@ -182,7 +182,7 @@ public class SSTableExportSchemaLoadingTest extends OfflineToolUtils
|
|||
*/
|
||||
private void assertPostTestEnv()
|
||||
{
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
assertKeyspaceNotLoaded();
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ public class SSTableExportTest extends OfflineToolUtils
|
|||
*/
|
||||
private void assertPostTestEnv()
|
||||
{
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ public class SSTableLevelResetterTest extends OfflineToolUtils
|
|||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
|
||||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
assertEquals(1, tool.getExitCode());
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ public class SSTableMetadataViewerTest extends OfflineToolUtils
|
|||
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Options:"));
|
||||
assertEquals(1, tool.getExitCode());
|
||||
}
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -218,7 +218,7 @@ public class SSTableMetadataViewerTest extends OfflineToolUtils
|
|||
|
||||
private void assertGoodEnvPostTest()
|
||||
{
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ public class SSTableOfflineRelevelTest extends OfflineToolUtils
|
|||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
|
||||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
assertEquals(1, tool.getExitCode());
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ public class SSTableRepairedAtSetterTest extends OfflineToolUtils
|
|||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
|
||||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
assertEquals(1, tool.getExitCode());
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -79,7 +79,7 @@ public class SSTableRepairedAtSetterTest extends OfflineToolUtils
|
|||
"--is-repaired",
|
||||
findOneSSTable("legacy_sstables", "legacy_ma_simple"));
|
||||
tool.assertOnCleanExit();
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -95,7 +95,7 @@ public class SSTableRepairedAtSetterTest extends OfflineToolUtils
|
|||
"--is-unrepaired",
|
||||
findOneSSTable("legacy_sstables", "legacy_ma_simple"));
|
||||
tool.assertOnCleanExit();
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -113,7 +113,7 @@ public class SSTableRepairedAtSetterTest extends OfflineToolUtils
|
|||
String file = tmpFile.absolutePath();
|
||||
ToolResult tool = ToolRunner.invokeClass(SSTableRepairedAtSetter.class, "--really-set", "--is-repaired", "-f", file);
|
||||
tool.assertOnCleanExit();
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA);
|
||||
assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ public class ToolsSchemaLoadingTest extends OfflineToolUtils
|
|||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
|
||||
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Missing arguments"));
|
||||
assertEquals(1, tool.getExitCode());
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -49,7 +49,7 @@ public class ToolsSchemaLoadingTest extends OfflineToolUtils
|
|||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
|
||||
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Missing arguments"));
|
||||
assertEquals(1, tool.getExitCode());
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -64,7 +64,7 @@ public class ToolsSchemaLoadingTest extends OfflineToolUtils
|
|||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
|
||||
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("No sstables to split"));
|
||||
assertEquals(1, tool.getExitCode());
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -79,7 +79,7 @@ public class ToolsSchemaLoadingTest extends OfflineToolUtils
|
|||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
|
||||
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Missing arguments"));
|
||||
assertEquals(1, tool.getExitCode());
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
@ -94,7 +94,7 @@ public class ToolsSchemaLoadingTest extends OfflineToolUtils
|
|||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
|
||||
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Missing arguments"));
|
||||
assertEquals(1, tool.getExitCode());
|
||||
assertNoUnexpectedThreadsStarted(null);
|
||||
assertNoUnexpectedThreadsStarted(null, false);
|
||||
assertSchemaNotLoaded();
|
||||
assertCLSMNotLoaded();
|
||||
assertSystemKSNotLoaded();
|
||||
|
|
|
|||
Loading…
Reference in New Issue