diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml index 35eb49206e..0b22f29349 100644 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@ -43,59 +43,77 @@ default_env_vars: &default_env_vars CCM_MAX_HEAP_SIZE: 1024M CCM_HEAP_NEWSIZE: 256M + # Whether the repeated test iterations should stop on the first failure by default. + REPEATED_TESTS_STOP_ON_FAILURE: false + + # Comma-separated list of tests that should be included in the repeated run for regular unit tests, + # in addition to automatically detected new and modified tests. For example: + # REPEATED_UTESTS: org.apache.cassandra.cql3.ViewTest + # REPEATED_UTESTS: org.apache.cassandra.cql3.ViewTest#testCountersTable + # REPEATED_UTESTS: org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.cql3.functions.TimeFctsTest + REPEATED_UTESTS: + # The number of times that new, modified or manually specified unit tests should be run. + REPEATED_UTESTS_COUNT: 500 + + # Comma-separated list of tests that should be included in the repeated run for long unit tests, + # in addition to automatically detected new and modified tests. For example: + # REPEATED_UTESTS_LONG: org.apache.cassandra.db.commitlog.CommitLogStressTest + # REPEATED_UTESTS_LONG: org.apache.cassandra.db.commitlog.CommitLogStressTest#testRandomSize + REPEATED_UTESTS_LONG: + # The number of times that new, modified or manually specified long unit tests should be run. + REPEATED_UTESTS_LONG_COUNT: 100 + + # Comma-separated list of tests that should be included in the repeated run for JVM dtests, + # in addition to automatically detected new and modified tests. For example: + # REPEATED_JVM_DTESTS: org.apache.cassandra.distributed.test.PagingTest + # REPEATED_JVM_DTESTS: org.apache.cassandra.distributed.test.PagingTest#testPaging + REPEATED_JVM_DTESTS: + # The number of times that new, modified or manually specified JVM dtests should be run. + REPEATED_JVM_DTESTS_COUNT: 500 + + # Comma-separated list of tests that should be included in the repeated run for JVM upgrade dtests, + # in addition to automatically detected new and modified tests. For example: + # REPEATED_JVM_UPGRADE_DTESTS: org.apache.cassandra.distributed.upgrade.GroupByTest + # REPEATED_JVM_UPGRADE_DTESTS: org.apache.cassandra.distributed.upgrade.GroupByTest#testReads + REPEATED_JVM_UPGRADE_DTESTS: + # The number of times that new, modified or manually specified JVM upgrade dtests should be run. + REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + + # Comma-separated list of Python dtests that should be repeatedly run, for example: + # REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py + # REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke + # REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke::test_create_index + # REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py,consistency_test.py + REPEATED_DTESTS: + # The number of times that the manually specified Python dtests should be run. + REPEATED_DTESTS_COUNT: 500 + + # Comma-separated list of Python dtests that should be repeatedly run, for example: + # REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py + # REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py::TestCQLNodes2RF1_Upgrade_current_4_0_x_To_indev_4_0_x + # REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py,upgrade_tests/paging_test.py + REPEATED_UPGRADE_DTESTS: + # The number of times that the manually specified Python upgrade dtests should be run. + REPEATED_UPGRADE_DTESTS_COUNT: 25 + # The Ant test target to run, for example: - # REPEATED_UTEST_TARGET: testsome - # REPEATED_UTEST_TARGET: test-jvm-dtest-some - # REPEATED_UTEST_TARGET: test-cdc - # REPEATED_UTEST_TARGET: test-compression - # REPEATED_UTEST_TARGET: test-system-keyspace-directory - REPEATED_UTEST_TARGET: testsome + # REPEATED_ANT_TEST_TARGET: testsome + # REPEATED_ANT_TEST_TARGET: test-jvm-dtest-some + # REPEATED_ANT_TEST_TARGET: test-cdc + # REPEATED_ANT_TEST_TARGET: test-compression + # REPEATED_ANT_TEST_TARGET: test-system-keyspace-directory + REPEATED_ANT_TEST_TARGET: testsome # The name of JUnit class to be run multiple times, for example: - # REPEATED_UTEST_CLASS: org.apache.cassandra.cql3.ViewTest - # REPEATED_UTEST_CLASS: org.apache.cassandra.distributed.test.PagingTest - REPEATED_UTEST_CLASS: - # The optional specific methods within REPEATED_UTEST_CLASS to be run, for example: - # REPEATED_UTEST_METHODS: testCompoundPartitionKey - # REPEATED_UTEST_METHODS: testCompoundPartitionKey,testStaticTable + # REPEATED_ANT_TEST_CLASS: org.apache.cassandra.cql3.ViewTest + # REPEATED_ANT_TEST_CLASS: org.apache.cassandra.distributed.test.PagingTest + REPEATED_ANT_TEST_CLASS: + # The optional specific methods within REPEATED_ANT_TEST_CLASS to be run, for example: + # REPEATED_ANT_TEST_METHODS: testCompoundPartitionKey + # REPEATED_ANT_TEST_METHODS: testCompoundPartitionKey,testStaticTable # Please note that some Ant targets will ignore the -Dtest.methods argument produced by this. - REPEATED_UTEST_METHODS: - # The number of times that the repeated JUnit test should be run - REPEATED_UTEST_COUNT: 100 - # Whether the test iteration should stop on the first failure - REPEATED_UTEST_STOP_ON_FAILURE: false - - # A Python dtest to be run multiple times, for example: - # REPEATED_DTEST_NAME: cqlsh_tests/test_cqlsh.py - # REPEATED_DTEST_NAME: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke - # REPEATED_DTEST_NAME: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke::test_create_index - REPEATED_DTEST_NAME: - # Whether the repeated Python dtest should use vnodes - REPEATED_DTEST_VNODES: false - # The number of times that the repeated Python dtest should be run - REPEATED_DTEST_COUNT: 100 - # Whether the test iteration should stop on the first failure - REPEATED_DTEST_STOP_ON_FAILURE: false - - # A Python upgrade dtest to be run multiple times, for example: - # REPEATED_UPGRADE_DTEST_NAME: upgrade_tests/cql_tests.py - # REPEATED_UPGRADE_DTEST_NAME: upgrade_tests/cql_tests.py::TestCQLNodes2RF1_Upgrade_current_4_0_x_To_indev_4_0_x - REPEATED_UPGRADE_DTEST_NAME: - # The number of times that the repeated Python upgrade dtest should be run - REPEATED_UPGRADE_DTEST_COUNT: 100 - # Whether the test iteration should stop on the first failure - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - # The name of JVM upgrade dtest class to be run multiple times, for example: - # REPEATED_JVM_UPGRADE_DTEST_CLASS: org.apache.cassandra.distributed.upgrade.MixedModeAvailabilityV30Test - REPEATED_JVM_UPGRADE_DTEST_CLASS: - # The optional specific methods within REPEATED_JVM_UPGRADE_DTEST_CLASS to be run, for example: - # REPEATED_JVM_UPGRADE_DTEST_METHODS: testAvailabilityV30ToV3X - # REPEATED_JVM_UPGRADE_DTEST_METHODS: testAvailabilityV30ToV3X,testAvailabilityV30ToV4 - REPEATED_JVM_UPGRADE_DTEST_METHODS: - # The number of times that the repeated JVM upgrade dtest should be run - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - # Whether the JVM upgrade dtest iteration should stop on the first failure - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + REPEATED_ANT_TEST_METHODS: + # The number of times that the repeated JUnit test should be run. + REPEATED_ANT_TEST_COUNT: 500 j8_par_executor: &j8_par_executor executor: @@ -141,7 +159,7 @@ j8_repeated_jvm_upgrade_dtest_executor: &j8_repeated_jvm_upgrade_dtest_executor name: java8-executor parallelism: 4 -with_dtests_jobs: &with_dtest_jobs +separate_jobs: &separate_jobs jobs: - start_build: type: approval @@ -155,12 +173,24 @@ with_dtests_jobs: &with_dtest_jobs requires: - start_j8_unit_tests - build + - start_j8_unit_tests_repeat: + type: approval + - j8_unit_tests_repeat: + requires: + - start_j8_unit_tests_repeat + - build - start_j8_jvm_dtests: type: approval - j8_jvm_dtests: requires: - start_j8_jvm_dtests - build + - start_j8_jvm_dtests_repeat: + type: approval + - j8_jvm_dtests_repeat: + requires: + - start_j8_jvm_dtests_repeat + - build # specialized unit tests (all run using Java 8) - start_utests_long: type: approval @@ -168,70 +198,88 @@ with_dtests_jobs: &with_dtest_jobs requires: - start_utests_long - build + - start_utests_long_repeat: + type: approval + - utests_long_repeat: + requires: + - start_utests_long_repeat + - build - start_utests_compression: type: approval - utests_compression: requires: - start_utests_compression - build + - start_utests_compression_repeat: + type: approval + - utests_compression_repeat: + requires: + - start_utests_compression_repeat + - build - start_j8_dtest_jars_build: type: approval - j8_dtest_jars_build: requires: - start_j8_dtest_jars_build - build - - start_jvm_upgrade_dtest: + - start_jvm_upgrade_dtests: type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest + - start_jvm_upgrade_dtests + - j8_dtest_jars_build + - start_jvm_upgrade_dtests_repeat: + type: approval + - j8_jvm_upgrade_dtests_repeat: + requires: + - start_jvm_upgrade_dtests_repeat - j8_dtest_jars_build # Java 8 dtests - start_j8_dtests: type: approval - - j8_dtests-with-vnodes: + - j8_dtests: requires: - start_j8_dtests - build - - j8_dtests-no-vnodes: + - start_j8_dtests_repeat: + type: approval + - j8_dtests_repeat: requires: - - start_j8_dtests + - start_j8_dtests_repeat + - build + - start_j8_dtests_vnode: + type: approval + - j8_dtests_vnode: + requires: + - start_j8_dtests_vnode + - build + - start_j8_dtests_vnode_repeat: + type: approval + - j8_dtests_vnode_repeat: + requires: + - start_j8_dtests_vnode_repeat - build # Java 8 upgrade tests - - start_upgrade_tests: + - start_j8_upgrade_dtests: type: approval - - j8_upgradetests-no-vnodes: + - j8_upgrade_dtests: requires: - - start_upgrade_tests + - start_j8_upgrade_dtests + - build + - start_j8_upgrade_dtests_repeat: + type: approval + - j8_upgrade_dtests_repeat: + requires: + - start_j8_upgrade_dtests_repeat - build # Java 8 repeated utest - - start_j8_repeated_utest: + - start_j8_repeated_ant_test: type: approval - - j8_repeated_utest: + - j8_repeated_ant_test: requires: - - start_j8_repeated_utest + - start_j8_repeated_ant_test - build - # Java 8 repeated dtest - - start_j8_repeated_dtest: - type: approval - - j8_repeated_dtest: - requires: - - start_j8_repeated_dtest - - build - # Repeated Python upgrade dtest - - start_repeated_upgrade_dtest: - type: approval - - repeated_upgrade_dtest: - requires: - - start_repeated_upgrade_dtest - - build - # Repeated JVM upgrade dtest - - start_repeated_jvm_upgrade_dtest: - type: approval - - repeated_jvm_upgrade_dtest: - requires: - - start_repeated_jvm_upgrade_dtest - - j8_dtest_jars_build + pre-commit_jobs: &pre-commit_jobs jobs: @@ -244,9 +292,15 @@ pre-commit_jobs: &pre-commit_jobs - j8_unit_tests: requires: - build + - j8_unit_tests_repeat: + requires: + - build - j8_jvm_dtests: requires: - build + - j8_jvm_dtests_repeat: + requires: + - build # specialized unit tests (all run on request using Java 8) - start_utests_long: type: approval @@ -254,70 +308,60 @@ pre-commit_jobs: &pre-commit_jobs requires: - start_utests_long - build + - utests_long_repeat: + requires: + - start_utests_long + - build - start_utests_compression: type: approval - utests_compression: requires: - start_utests_compression - build - - start_j8_dtest_jars_build: + - utests_compression_repeat: + requires: + - start_utests_compression + - build + - start_jvm_upgrade_dtests: type: approval - j8_dtest_jars_build: requires: - - start_j8_dtest_jars_build + - start_jvm_upgrade_dtests - build - - start_jvm_upgrade_dtest: - type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest + - j8_dtest_jars_build + - j8_jvm_upgrade_dtests_repeat: + requires: - j8_dtest_jars_build # Java 8 dtests - - j8_dtests-with-vnodes: + - j8_dtests: requires: - build - - j8_dtests-no-vnodes: + - j8_dtests_repeat: + requires: + - build + - j8_dtests_vnode: + requires: + - build + - j8_dtests_vnode_repeat: requires: - build # Java 8 upgrade tests (on request) - - start_upgrade_tests: + - start_upgrade_dtests: type: approval - - j8_upgradetests-no-vnodes: + - j8_upgrade_dtests: requires: - - start_upgrade_tests + - start_upgrade_dtests - build - # Java 8 repeated utest (on request) - - start_j8_repeated_utest: - type: approval - - j8_repeated_utest: + - j8_upgrade_dtests_repeat: requires: - - start_j8_repeated_utest + - start_upgrade_dtests - build - # Java 8 repeated dtest (on request) - - start_j8_repeated_dtest: - type: approval - - j8_repeated_dtest: - requires: - - start_j8_repeated_dtest - - build - # Repeated Python upgrade dtest (on request) - - start_repeated_upgrade_dtest: - type: approval - - repeated_upgrade_dtest: - requires: - - start_repeated_upgrade_dtest - - build - # Repeated JVM upgrade dtest (on request) - - start_repeated_jvm_upgrade_dtest: - type: approval - - repeated_jvm_upgrade_dtest: - requires: - - start_repeated_jvm_upgrade_dtest - - j8_dtest_jars_build workflows: version: 2 - separate_tests: *with_dtest_jobs + separate_tests: *separate_jobs pre-commit_tests: *pre-commit_jobs executors: @@ -414,7 +458,7 @@ jobs: - run_parallel_junit_tests: target: testclasslist-compression - j8_dtests-with-vnodes: + j8_dtests_vnode: <<: *j8_par_executor steps: - attach_workspace: @@ -428,7 +472,7 @@ jobs: file_tag: j8_with_vnodes pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' - j8_dtests-no-vnodes: + j8_dtests: <<: *j8_par_executor steps: - attach_workspace: @@ -442,7 +486,7 @@ jobs: file_tag: j8_without_vnodes pytest_extra_args: '--skip-resource-intensive-tests' - j8_upgradetests-no-vnodes: + j8_upgrade_dtests: <<: *j8_par_executor steps: - attach_workspace: @@ -459,20 +503,68 @@ jobs: extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' pytest_extra_args: '--execute-upgrade-tests' - j8_repeated_utest: + j8_unit_tests_repeat: + <<: *j8_repeated_utest_executor + steps: + - attach_workspace: + at: /home/cassandra + - log_environment + - run_unit_tests_repeat + + utests_compression_repeat: + <<: *j8_repeated_utest_executor + steps: + - attach_workspace: + at: /home/cassandra + - log_environment + - run_utests_compression_repeat + + utests_system_keyspace_directory_repeat: + <<: *j8_repeated_utest_executor + steps: + - attach_workspace: + at: /home/cassandra + - log_environment + - run_utests_system_keyspace_directory_repeat + + utests_long_repeat: + <<: *j8_repeated_utest_executor + steps: + - attach_workspace: + at: /home/cassandra + - log_environment + - run_utests_long_repeat + + j8_jvm_dtests_repeat: + <<: *j8_repeated_utest_executor + steps: + - attach_workspace: + at: /home/cassandra + - log_environment + - run_jvm_dtests_repeat + + j8_jvm_upgrade_dtests_repeat: + <<: *j8_repeated_jvm_upgrade_dtest_executor + steps: + - attach_workspace: + at: /home/cassandra + - log_environment + - run_jvm_upgrade_dtests_repeat + + j8_repeated_ant_test: <<: *j8_repeated_utest_executor steps: - attach_workspace: at: /home/cassandra - log_environment - run_repeated_utest: - target: ${REPEATED_UTEST_TARGET} - class: ${REPEATED_UTEST_CLASS} - methods: ${REPEATED_UTEST_METHODS} - count: ${REPEATED_UTEST_COUNT} - stop_on_failure: ${REPEATED_UTEST_STOP_ON_FAILURE} + target: ${REPEATED_ANT_TEST_TARGET} + class: ${REPEATED_ANT_TEST_CLASS} + methods: ${REPEATED_ANT_TEST_METHODS} + count: ${REPEATED_ANT_TEST_COUNT} + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} - j8_repeated_dtest: + j8_dtests_repeat: <<: *j8_repeated_dtest_executor steps: - attach_workspace: @@ -480,26 +572,27 @@ jobs: - clone_dtest - create_venv - run_repeated_dtest: - tests: ${REPEATED_DTEST_NAME} - vnodes: ${REPEATED_DTEST_VNODES} + tests: ${REPEATED_DTESTS} + vnodes: "false" upgrade: "false" - count: ${REPEATED_DTEST_COUNT} - stop_on_failure: ${REPEATED_DTEST_STOP_ON_FAILURE} + count: ${REPEATED_DTESTS_COUNT} + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} - repeated_jvm_upgrade_dtest: - <<: *j8_repeated_jvm_upgrade_dtest_executor + j8_dtests_vnode_repeat: + <<: *j8_repeated_dtest_executor steps: - attach_workspace: at: /home/cassandra - - log_environment - - run_repeated_utest: - target: test-jvm-dtest-some - class: ${REPEATED_JVM_UPGRADE_DTEST_CLASS} - methods: ${REPEATED_JVM_UPGRADE_DTEST_METHODS} - count: ${REPEATED_JVM_UPGRADE_DTEST_COUNT} - stop_on_failure: ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} + - clone_dtest + - create_venv + - run_repeated_dtest: + tests: ${REPEATED_DTESTS} + vnodes: "true" + upgrade: "false" + count: ${REPEATED_DTESTS_COUNT} + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} - repeated_upgrade_dtest: + j8_upgrade_dtests_repeat: <<: *j8_repeated_upgrade_dtest_executor steps: - attach_workspace: @@ -507,11 +600,11 @@ jobs: - clone_dtest - create_venv - run_repeated_dtest: - tests: ${REPEATED_UPGRADE_DTEST_NAME} + tests: ${REPEATED_UPGRADE_DTESTS} vnodes: "false" upgrade: "true" - stop_on_failure: ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE} - count: ${REPEATED_UPGRADE_DTEST_COUNT} + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} + count: ${REPEATED_UPGRADE_DTESTS_COUNT} commands: log_environment: @@ -831,6 +924,172 @@ commands: path: ~/cassandra-dtest/logs destination: dtest_<>_logs + run_unit_tests_repeat: + steps: + - run_repeated_utests: + target: testsome + tests: ${REPEATED_UTESTS} + count: ${REPEATED_UTESTS_COUNT} + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} + + run_utests_compression_repeat: + steps: + - run_repeated_utests: + target: test-compression + tests: ${REPEATED_UTESTS} + count: ${REPEATED_UTESTS_COUNT} + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} + + run_utests_system_keyspace_directory_repeat: + steps: + - run_repeated_utests: + target: test-system-keyspace-directory + tests: ${REPEATED_UTESTS} + count: ${REPEATED_UTESTS_COUNT} + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} + + run_utests_long_repeat: + steps: + - run_repeated_utests: + target: long-testsome + tests: ${REPEATED_UTESTS_LONG} + count: ${REPEATED_UTESTS_LONG_COUNT} + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} + + run_jvm_dtests_repeat: + steps: + - run_repeated_utests: + target: test-jvm-dtest-some + tests: ${REPEATED_JVM_DTESTS} + count: ${REPEATED_JVM_DTESTS_COUNT} + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} + + run_jvm_upgrade_dtests_repeat: + steps: + - run_repeated_utests: + target: test-jvm-dtest-some + tests: ${REPEATED_JVM_UPGRADE_DTESTS} + count: ${REPEATED_JVM_UPGRADE_DTESTS_COUNT} + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} + + run_repeated_utests: + parameters: + target: + type: string + tests: + type: string + count: + type: string + stop_on_failure: + type: string + steps: + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((<> / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (<> % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo <> | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant <> $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ <> = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + run_repeated_utest: parameters: target: @@ -885,7 +1144,8 @@ commands: if [[ $target == "test" || \ $target == "test-cdc" || \ $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then name="-Dtest.name=$class_name" else name="-Dtest.name=$class_path" @@ -1002,6 +1262,8 @@ commands: echo "** done env" mkdir -p /tmp/results/dtests + tests_arg=$(echo <> | sed -e "s/,/ /g") + stop_on_failure_arg="" if <>; then stop_on_failure_arg="-x" @@ -1014,11 +1276,11 @@ commands: upgrade_arg="" if <>; then - upgrade_arg="--execute-upgrade-tests" + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" fi # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir <> | tee /tmp/dtest/stdout.txt + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt fi fi - store_test_results: diff --git a/.circleci/config-2_1.yml.high_res.patch b/.circleci/config-2_1.yml.high_res.patch index 9bb0d6104e..bdc37a0307 100644 --- a/.circleci/config-2_1.yml.high_res.patch +++ b/.circleci/config-2_1.yml.high_res.patch @@ -1,5 +1,5 @@ ---- .circleci/config-2_1.yml 2021-10-19 11:08:44.000000000 +0100 -+++ .circleci/config-2_1.yml.HIGHRES 2021-10-19 11:26:16.000000000 +0100 +--- config-2_1.yml 2022-10-15 16:29:48.670020291 +0100 ++++ config-2_1.yml.HIGHRES 2022-10-15 16:30:17.885329362 +0100 @@ -40,8 +40,8 @@ CASSANDRA_SKIP_SYNC: true DTEST_REPO: https://github.com/apache/cassandra-dtest.git @@ -9,9 +9,9 @@ + CCM_MAX_HEAP_SIZE: 2048M + CCM_HEAP_NEWSIZE: 512M - # The Ant test target to run, for example: - # REPEATED_UTEST_TARGET: testsome -@@ -100,46 +100,50 @@ + # Whether the repeated test iterations should stop on the first failure by default. + REPEATED_TESTS_STOP_ON_FAILURE: false +@@ -118,46 +118,50 @@ j8_par_executor: &j8_par_executor executor: name: java8-executor @@ -71,5 +71,5 @@ + exec_resource_class: xlarge + parallelism: 100 - with_dtests_jobs: &with_dtest_jobs + separate_jobs: &separate_jobs jobs: diff --git a/.circleci/config-2_1.yml.mid_res.patch b/.circleci/config-2_1.yml.mid_res.patch index bcb47fcaff..05cf0c275e 100644 --- a/.circleci/config-2_1.yml.mid_res.patch +++ b/.circleci/config-2_1.yml.mid_res.patch @@ -1,6 +1,6 @@ ---- .circleci/config-2_1.yml 2021-10-19 11:08:44.000000000 +0100 -+++ .circleci/config-2_1.yml.MIDRES 2021-10-19 11:26:16.000000000 +0100 -@@ -101,45 +101,65 @@ +--- config-2_1.yml 2022-10-15 16:29:48.670020291 +0100 ++++ config-2_1.yml.MIDRES 2022-10-15 16:30:17.879007207 +0100 +@@ -119,45 +119,65 @@ executor: name: java8-executor #exec_resource_class: xlarge @@ -71,30 +71,30 @@ + exec_resource_class: medium + parallelism: 25 - with_dtests_jobs: &with_dtest_jobs + separate_jobs: &separate_jobs jobs: -@@ -415,7 +435,7 @@ +@@ -459,7 +479,7 @@ target: testclasslist-compression - j8_dtests-with-vnodes: + j8_dtests_vnode: - <<: *j8_par_executor + <<: *j8_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -429,7 +449,7 @@ +@@ -473,7 +493,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' - j8_dtests-no-vnodes: + j8_dtests: - <<: *j8_par_executor + <<: *j8_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -443,7 +463,7 @@ +@@ -487,7 +507,7 @@ pytest_extra_args: '--skip-resource-intensive-tests' - j8_upgradetests-no-vnodes: + j8_upgrade_dtests: - <<: *j8_par_executor + <<: *j8_very_large_par_executor steps: diff --git a/.circleci/config.yml b/.circleci/config.yml index a3fd9e1a2d..43a507ba9f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,25 +108,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - repeated_jvm_upgrade_dtest: + utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -159,81 +160,80 @@ jobs: which java java -version - run: - name: Run repeated JUnit test + name: Repeatedly run new or modifed JUnit tests no_output_timeout: 15m command: | - if [ "${REPEATED_JVM_UPGRADE_DTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_JVM_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running test-jvm-dtest-some ${REPEATED_JVM_UPGRADE_DTEST_CLASS} ${REPEATED_JVM_UPGRADE_DTEST_METHODS} ${REPEATED_JVM_UPGRADE_DTEST_COUNT} times" + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" fi - target=test-jvm-dtest-some - class_path=${REPEATED_JVM_UPGRADE_DTEST_CLASS} - class_name="${class_path##*.}" - # Prepare the -Dtest.name argument. # It can be the fully qualified class name or the short class name, depending on the target. if [[ $target == "test" || \ $target == "test-cdc" || \ $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" else - name="-Dtest.name=$class_path" + name_arg="-Dtest.name=$class" fi # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_JVM_UPGRADE_DTEST_METHODS}" == "" ]; then - methods="" + if [ $method == "" ]; then + methods_arg="" else - methods="-Dtest.methods=${REPEATED_JVM_UPGRADE_DTEST_METHODS}" + methods_arg="-Dtest.methods=$method" fi - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" + echo "Running test $test, iteration $i of $count" # run the test status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + if !( set -o pipefail && \ + ant test-compression $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then status="fails" exit_code=1 fi # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} + dest=/tmp/results/repeated_utests/stdout/${status}/${i} mkdir -p $dest - mv stdout.txt $dest/test-jvm-dtest-some-${REPEATED_JVM_UPGRADE_DTEST_CLASS}.txt + mv stdout.txt $dest/${test}.txt # move the XML output files source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} + dest=/tmp/results/repeated_utests/output/${status}/${i} mkdir -p $dest if [[ -d $source && -n "$(ls $source)" ]]; then mv $source/* $dest/ @@ -241,31 +241,29 @@ jobs: # move the log files source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} + dest=/tmp/results/repeated_utests/logs/${status}/${i} mkdir -p $dest if [[ -d $source && -n "$(ls $source)" ]]; then mv $source/* $dest/ fi # maybe stop iterations on test failure - if [[ ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then break fi done - - (exit ${exit_code}) - fi - fi + done + (exit ${exit_code}) - store_test_results: - path: /tmp/results/repeated_utest/output + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/results/repeated_utest/stdout + path: /tmp/results/repeated_utests/stdout destination: stdout - store_artifacts: - path: /tmp/results/repeated_utest/output + path: /tmp/results/repeated_utests/output destination: junitxml - store_artifacts: - path: /tmp/results/repeated_utest/logs + path: /tmp/results/repeated_utests/logs destination: logs environment: - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 @@ -282,25 +280,370 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - repeated_upgrade_dtest: + utests_long_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_unit_tests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant testsome $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_upgrade_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -329,25 +672,25 @@ jobs: name: Run repeated Python dtest no_output_timeout: 15m command: | - if [ "${REPEATED_UPGRADE_DTEST_NAME}" == "" ]; then + if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" == "" ]; then + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" -le 0 ]; then + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" else # Calculate the number of test iterations to be run by the current parallel runner. # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then count=$((count+1)) fi if (($count <= 0)); then echo "No tests to run in this runner" else - echo "Running ${REPEATED_UPGRADE_DTEST_NAME} $count times" + echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" source ~/env3.6/bin/activate export PATH=$JAVA_HOME/bin:$PATH @@ -360,8 +703,10 @@ jobs: echo "** done env" mkdir -p /tmp/results/dtests + tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") + stop_on_failure_arg="" - if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then stop_on_failure_arg="-x" fi @@ -372,11 +717,11 @@ jobs: upgrade_arg="" if true; then - upgrade_arg="--execute-upgrade-tests" + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" fi # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} | tee /tmp/dtest/stdout.txt + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt fi fi - store_test_results: @@ -402,142 +747,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_repeated_dtest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Run repeated Python dtest - no_output_timeout: 15m - command: | - if [ "${REPEATED_DTEST_NAME}" == "" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" == "" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_DTEST_NAME} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - stop_on_failure_arg="" - if ${REPEATED_DTEST_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if ${REPEATED_DTEST_VNODES}; then - vnodes_args="--use-vnodes --num-tokens=16" - fi - - upgrade_arg="" - if false; then - upgrade_arg="--execute-upgrade-tests" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: @@ -619,25 +845,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-no-vnodes: + j8_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -663,21 +890,69 @@ jobs: pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt pip3 freeze - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - - run: - name: Run dtests (j8_without_vnodes) + name: Run repeated Python dtest no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: | + if [ "${REPEATED_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if true; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi - store_test_results: path: /tmp/results - store_artifacts: path: /tmp/dtest - destination: dtest_j8_without_vnodes + destination: dtest - store_artifacts: path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + destination: dtest_logs environment: - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - ANT_HOME: /usr/share/ant @@ -693,25 +968,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgradetests-no-vnodes: + j8_upgrade_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -808,22 +1084,195 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_unit_tests: @@ -916,25 +1365,101 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-with-vnodes: + j8_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -990,22 +1515,494 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated_ant_test: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run repeated JUnit test + no_output_timeout: 15m + command: | + if [ "${REPEATED_ANT_TEST_CLASS}" == "" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_ANT_TEST_COUNT}" == "" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=${REPEATED_ANT_TEST_TARGET} + class_path=${REPEATED_ANT_TEST_CLASS} + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "${REPEATED_ANT_TEST_METHODS}" == "" ]; then + methods="" + else + methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if stop_on_failure is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Run repeated Python dtest + no_output_timeout: 15m + command: | + if [ "${REPEATED_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_jvm_dtests: @@ -1098,22 +2095,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_long: @@ -1160,22 +2158,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_compression: @@ -1268,196 +2267,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_repeated_utest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run repeated JUnit test - no_output_timeout: 15m - command: | - if [ "${REPEATED_UTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times" - - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - target=${REPEATED_UTEST_TARGET} - class_path=${REPEATED_UTEST_CLASS} - class_name="${class_path##*.}" - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" - else - name="-Dtest.name=$class_path" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_UTEST_METHODS}" == "" ]; then - methods="" - else - methods="-Dtest.methods=${REPEATED_UTEST_METHODS}" - fi - - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" - for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt - - # move the XML output files - source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - - (exit ${exit_code}) - fi - fi - - store_test_results: - path: /tmp/results/repeated_utest/output - - store_artifacts: - path: /tmp/results/repeated_utest/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utest/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utest/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtest_jars_build: @@ -1536,22 +2362,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: @@ -1593,52 +2420,30 @@ workflows: requires: - start_j8_dtest_jars_build - build - - start_jvm_upgrade_dtest: + - start_jvm_upgrade_dtests: type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest + - start_jvm_upgrade_dtests - j8_dtest_jars_build - start_j8_dtests: type: approval - - j8_dtests-with-vnodes: + - j8_dtests: requires: - start_j8_dtests - build - - j8_dtests-no-vnodes: - requires: - - start_j8_dtests - - build - - start_upgrade_tests: + - start_j8_dtests_vnode: type: approval - - j8_upgradetests-no-vnodes: + - j8_dtests_vnode: requires: - - start_upgrade_tests + - start_j8_dtests_vnode - build - - start_j8_repeated_utest: + - start_j8_upgrade_dtests: type: approval - - j8_repeated_utest: + - j8_upgrade_dtests: requires: - - start_j8_repeated_utest + - start_j8_upgrade_dtests - build - - start_j8_repeated_dtest: - type: approval - - j8_repeated_dtest: - requires: - - start_j8_repeated_dtest - - build - - start_repeated_upgrade_dtest: - type: approval - - repeated_upgrade_dtest: - requires: - - start_repeated_upgrade_dtest - - build - - start_repeated_jvm_upgrade_dtest: - type: approval - - repeated_jvm_upgrade_dtest: - requires: - - start_repeated_jvm_upgrade_dtest - - j8_dtest_jars_build pre-commit_tests: jobs: - start_pre-commit_tests: @@ -1664,51 +2469,24 @@ workflows: requires: - start_utests_compression - build - - start_j8_dtest_jars_build: + - start_jvm_upgrade_dtests: type: approval - j8_dtest_jars_build: requires: - - start_j8_dtest_jars_build + - start_jvm_upgrade_dtests - build - - start_jvm_upgrade_dtest: - type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest - j8_dtest_jars_build - - j8_dtests-with-vnodes: + - j8_dtests: requires: - build - - j8_dtests-no-vnodes: + - j8_dtests_vnode: requires: - build - - start_upgrade_tests: + - start_upgrade_dtests: type: approval - - j8_upgradetests-no-vnodes: + - j8_upgrade_dtests: requires: - - start_upgrade_tests + - start_upgrade_dtests - build - - start_j8_repeated_utest: - type: approval - - j8_repeated_utest: - requires: - - start_j8_repeated_utest - - build - - start_j8_repeated_dtest: - type: approval - - j8_repeated_dtest: - requires: - - start_j8_repeated_dtest - - build - - start_repeated_upgrade_dtest: - type: approval - - repeated_upgrade_dtest: - requires: - - start_repeated_upgrade_dtest - - build - - start_repeated_jvm_upgrade_dtest: - type: approval - - repeated_jvm_upgrade_dtest: - requires: - - start_repeated_jvm_upgrade_dtest - - j8_dtest_jars_build diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES index 7a93ab4b3d..b4983f9b8a 100644 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@ -108,25 +108,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - repeated_jvm_upgrade_dtest: + utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: xlarge @@ -159,81 +160,80 @@ jobs: which java java -version - run: - name: Run repeated JUnit test + name: Repeatedly run new or modifed JUnit tests no_output_timeout: 15m command: | - if [ "${REPEATED_JVM_UPGRADE_DTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_JVM_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running test-jvm-dtest-some ${REPEATED_JVM_UPGRADE_DTEST_CLASS} ${REPEATED_JVM_UPGRADE_DTEST_METHODS} ${REPEATED_JVM_UPGRADE_DTEST_COUNT} times" + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" fi - target=test-jvm-dtest-some - class_path=${REPEATED_JVM_UPGRADE_DTEST_CLASS} - class_name="${class_path##*.}" - # Prepare the -Dtest.name argument. # It can be the fully qualified class name or the short class name, depending on the target. if [[ $target == "test" || \ $target == "test-cdc" || \ $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" else - name="-Dtest.name=$class_path" + name_arg="-Dtest.name=$class" fi # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_JVM_UPGRADE_DTEST_METHODS}" == "" ]; then - methods="" + if [ $method == "" ]; then + methods_arg="" else - methods="-Dtest.methods=${REPEATED_JVM_UPGRADE_DTEST_METHODS}" + methods_arg="-Dtest.methods=$method" fi - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" + echo "Running test $test, iteration $i of $count" # run the test status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + if !( set -o pipefail && \ + ant test-compression $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then status="fails" exit_code=1 fi # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} + dest=/tmp/results/repeated_utests/stdout/${status}/${i} mkdir -p $dest - mv stdout.txt $dest/test-jvm-dtest-some-${REPEATED_JVM_UPGRADE_DTEST_CLASS}.txt + mv stdout.txt $dest/${test}.txt # move the XML output files source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} + dest=/tmp/results/repeated_utests/output/${status}/${i} mkdir -p $dest if [[ -d $source && -n "$(ls $source)" ]]; then mv $source/* $dest/ @@ -241,31 +241,29 @@ jobs: # move the log files source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} + dest=/tmp/results/repeated_utests/logs/${status}/${i} mkdir -p $dest if [[ -d $source && -n "$(ls $source)" ]]; then mv $source/* $dest/ fi # maybe stop iterations on test failure - if [[ ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then break fi done - - (exit ${exit_code}) - fi - fi + done + (exit ${exit_code}) - store_test_results: - path: /tmp/results/repeated_utest/output + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/results/repeated_utest/stdout + path: /tmp/results/repeated_utests/stdout destination: stdout - store_artifacts: - path: /tmp/results/repeated_utest/output + path: /tmp/results/repeated_utests/output destination: junitxml - store_artifacts: - path: /tmp/results/repeated_utest/logs + path: /tmp/results/repeated_utests/logs destination: logs environment: - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 @@ -282,25 +280,370 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - repeated_upgrade_dtest: + utests_long_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_unit_tests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant testsome $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_upgrade_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: xlarge @@ -329,25 +672,25 @@ jobs: name: Run repeated Python dtest no_output_timeout: 15m command: | - if [ "${REPEATED_UPGRADE_DTEST_NAME}" == "" ]; then + if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" == "" ]; then + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" -le 0 ]; then + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" else # Calculate the number of test iterations to be run by the current parallel runner. # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then count=$((count+1)) fi if (($count <= 0)); then echo "No tests to run in this runner" else - echo "Running ${REPEATED_UPGRADE_DTEST_NAME} $count times" + echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" source ~/env3.6/bin/activate export PATH=$JAVA_HOME/bin:$PATH @@ -360,8 +703,10 @@ jobs: echo "** done env" mkdir -p /tmp/results/dtests + tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") + stop_on_failure_arg="" - if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then stop_on_failure_arg="-x" fi @@ -372,11 +717,11 @@ jobs: upgrade_arg="" if true; then - upgrade_arg="--execute-upgrade-tests" + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" fi # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} | tee /tmp/dtest/stdout.txt + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt fi fi - store_test_results: @@ -402,142 +747,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_repeated_dtest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Run repeated Python dtest - no_output_timeout: 15m - command: | - if [ "${REPEATED_DTEST_NAME}" == "" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" == "" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_DTEST_NAME} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - stop_on_failure_arg="" - if ${REPEATED_DTEST_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if ${REPEATED_DTEST_VNODES}; then - vnodes_args="--use-vnodes --num-tokens=16" - fi - - upgrade_arg="" - if false; then - upgrade_arg="--execute-upgrade-tests" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: @@ -619,25 +845,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-no-vnodes: + j8_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: xlarge @@ -663,21 +890,69 @@ jobs: pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt pip3 freeze - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - - run: - name: Run dtests (j8_without_vnodes) + name: Run repeated Python dtest no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: | + if [ "${REPEATED_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if true; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi - store_test_results: path: /tmp/results - store_artifacts: path: /tmp/dtest - destination: dtest_j8_without_vnodes + destination: dtest - store_artifacts: path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + destination: dtest_logs environment: - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - ANT_HOME: /usr/share/ant @@ -693,25 +968,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgradetests-no-vnodes: + j8_upgrade_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: xlarge @@ -808,22 +1084,195 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_unit_tests: @@ -916,25 +1365,101 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-with-vnodes: + j8_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: xlarge @@ -990,22 +1515,494 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated_ant_test: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run repeated JUnit test + no_output_timeout: 15m + command: | + if [ "${REPEATED_ANT_TEST_CLASS}" == "" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_ANT_TEST_COUNT}" == "" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=${REPEATED_ANT_TEST_TARGET} + class_path=${REPEATED_ANT_TEST_CLASS} + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "${REPEATED_ANT_TEST_METHODS}" == "" ]; then + methods="" + else + methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if stop_on_failure is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Run repeated Python dtest + no_output_timeout: 15m + command: | + if [ "${REPEATED_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_jvm_dtests: @@ -1098,22 +2095,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_long: @@ -1160,22 +2158,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_compression: @@ -1268,196 +2267,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_repeated_utest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run repeated JUnit test - no_output_timeout: 15m - command: | - if [ "${REPEATED_UTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times" - - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - target=${REPEATED_UTEST_TARGET} - class_path=${REPEATED_UTEST_CLASS} - class_name="${class_path##*.}" - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" - else - name="-Dtest.name=$class_path" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_UTEST_METHODS}" == "" ]; then - methods="" - else - methods="-Dtest.methods=${REPEATED_UTEST_METHODS}" - fi - - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" - for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt - - # move the XML output files - source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - - (exit ${exit_code}) - fi - fi - - store_test_results: - path: /tmp/results/repeated_utest/output - - store_artifacts: - path: /tmp/results/repeated_utest/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utest/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utest/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtest_jars_build: @@ -1536,22 +2362,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 2048M - CCM_HEAP_NEWSIZE: 512M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: @@ -1569,76 +2396,108 @@ workflows: requires: - start_j8_unit_tests - build + - start_j8_unit_tests_repeat: + type: approval + - j8_unit_tests_repeat: + requires: + - start_j8_unit_tests_repeat + - build - start_j8_jvm_dtests: type: approval - j8_jvm_dtests: requires: - start_j8_jvm_dtests - build + - start_j8_jvm_dtests_repeat: + type: approval + - j8_jvm_dtests_repeat: + requires: + - start_j8_jvm_dtests_repeat + - build - start_utests_long: type: approval - utests_long: requires: - start_utests_long - build + - start_utests_long_repeat: + type: approval + - utests_long_repeat: + requires: + - start_utests_long_repeat + - build - start_utests_compression: type: approval - utests_compression: requires: - start_utests_compression - build + - start_utests_compression_repeat: + type: approval + - utests_compression_repeat: + requires: + - start_utests_compression_repeat + - build - start_j8_dtest_jars_build: type: approval - j8_dtest_jars_build: requires: - start_j8_dtest_jars_build - build - - start_jvm_upgrade_dtest: + - start_jvm_upgrade_dtests: type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest + - start_jvm_upgrade_dtests + - j8_dtest_jars_build + - start_jvm_upgrade_dtests_repeat: + type: approval + - j8_jvm_upgrade_dtests_repeat: + requires: + - start_jvm_upgrade_dtests_repeat - j8_dtest_jars_build - start_j8_dtests: type: approval - - j8_dtests-with-vnodes: + - j8_dtests: requires: - start_j8_dtests - build - - j8_dtests-no-vnodes: - requires: - - start_j8_dtests - - build - - start_upgrade_tests: + - start_j8_dtests_repeat: type: approval - - j8_upgradetests-no-vnodes: + - j8_dtests_repeat: requires: - - start_upgrade_tests + - start_j8_dtests_repeat - build - - start_j8_repeated_utest: + - start_j8_dtests_vnode: type: approval - - j8_repeated_utest: + - j8_dtests_vnode: requires: - - start_j8_repeated_utest + - start_j8_dtests_vnode - build - - start_j8_repeated_dtest: + - start_j8_dtests_vnode_repeat: type: approval - - j8_repeated_dtest: + - j8_dtests_vnode_repeat: requires: - - start_j8_repeated_dtest + - start_j8_dtests_vnode_repeat - build - - start_repeated_upgrade_dtest: + - start_j8_upgrade_dtests: type: approval - - repeated_upgrade_dtest: + - j8_upgrade_dtests: requires: - - start_repeated_upgrade_dtest + - start_j8_upgrade_dtests - build - - start_repeated_jvm_upgrade_dtest: + - start_j8_upgrade_dtests_repeat: type: approval - - repeated_jvm_upgrade_dtest: + - j8_upgrade_dtests_repeat: requires: - - start_repeated_jvm_upgrade_dtest - - j8_dtest_jars_build + - start_j8_upgrade_dtests_repeat + - build + - start_j8_repeated_ant_test: + type: approval + - j8_repeated_ant_test: + requires: + - start_j8_repeated_ant_test + - build pre-commit_tests: jobs: - start_pre-commit_tests: @@ -1649,66 +2508,66 @@ workflows: - j8_unit_tests: requires: - build + - j8_unit_tests_repeat: + requires: + - build - j8_jvm_dtests: requires: - build + - j8_jvm_dtests_repeat: + requires: + - build - start_utests_long: type: approval - utests_long: requires: - start_utests_long - build + - utests_long_repeat: + requires: + - start_utests_long + - build - start_utests_compression: type: approval - utests_compression: requires: - start_utests_compression - build - - start_j8_dtest_jars_build: + - utests_compression_repeat: + requires: + - start_utests_compression + - build + - start_jvm_upgrade_dtests: type: approval - j8_dtest_jars_build: requires: - - start_j8_dtest_jars_build + - start_jvm_upgrade_dtests - build - - start_jvm_upgrade_dtest: - type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest - j8_dtest_jars_build - - j8_dtests-with-vnodes: + - j8_jvm_upgrade_dtests_repeat: requires: - - build - - j8_dtests-no-vnodes: - requires: - - build - - start_upgrade_tests: - type: approval - - j8_upgradetests-no-vnodes: - requires: - - start_upgrade_tests - - build - - start_j8_repeated_utest: - type: approval - - j8_repeated_utest: - requires: - - start_j8_repeated_utest - - build - - start_j8_repeated_dtest: - type: approval - - j8_repeated_dtest: - requires: - - start_j8_repeated_dtest - - build - - start_repeated_upgrade_dtest: - type: approval - - repeated_upgrade_dtest: - requires: - - start_repeated_upgrade_dtest - - build - - start_repeated_jvm_upgrade_dtest: - type: approval - - repeated_jvm_upgrade_dtest: - requires: - - start_repeated_jvm_upgrade_dtest - j8_dtest_jars_build + - j8_dtests: + requires: + - build + - j8_dtests_repeat: + requires: + - build + - j8_dtests_vnode: + requires: + - build + - j8_dtests_vnode_repeat: + requires: + - build + - start_upgrade_dtests: + type: approval + - j8_upgrade_dtests: + requires: + - start_upgrade_dtests + - build + - j8_upgrade_dtests_repeat: + requires: + - start_upgrade_dtests + - build diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES index a3fd9e1a2d..baeb67190a 100644 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@ -108,25 +108,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - repeated_jvm_upgrade_dtest: + utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -159,81 +160,80 @@ jobs: which java java -version - run: - name: Run repeated JUnit test + name: Repeatedly run new or modifed JUnit tests no_output_timeout: 15m command: | - if [ "${REPEATED_JVM_UPGRADE_DTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_JVM_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running test-jvm-dtest-some ${REPEATED_JVM_UPGRADE_DTEST_CLASS} ${REPEATED_JVM_UPGRADE_DTEST_METHODS} ${REPEATED_JVM_UPGRADE_DTEST_COUNT} times" + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" fi - target=test-jvm-dtest-some - class_path=${REPEATED_JVM_UPGRADE_DTEST_CLASS} - class_name="${class_path##*.}" - # Prepare the -Dtest.name argument. # It can be the fully qualified class name or the short class name, depending on the target. if [[ $target == "test" || \ $target == "test-cdc" || \ $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" else - name="-Dtest.name=$class_path" + name_arg="-Dtest.name=$class" fi # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_JVM_UPGRADE_DTEST_METHODS}" == "" ]; then - methods="" + if [ $method == "" ]; then + methods_arg="" else - methods="-Dtest.methods=${REPEATED_JVM_UPGRADE_DTEST_METHODS}" + methods_arg="-Dtest.methods=$method" fi - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" + echo "Running test $test, iteration $i of $count" # run the test status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + if !( set -o pipefail && \ + ant test-compression $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then status="fails" exit_code=1 fi # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} + dest=/tmp/results/repeated_utests/stdout/${status}/${i} mkdir -p $dest - mv stdout.txt $dest/test-jvm-dtest-some-${REPEATED_JVM_UPGRADE_DTEST_CLASS}.txt + mv stdout.txt $dest/${test}.txt # move the XML output files source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} + dest=/tmp/results/repeated_utests/output/${status}/${i} mkdir -p $dest if [[ -d $source && -n "$(ls $source)" ]]; then mv $source/* $dest/ @@ -241,31 +241,29 @@ jobs: # move the log files source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} + dest=/tmp/results/repeated_utests/logs/${status}/${i} mkdir -p $dest if [[ -d $source && -n "$(ls $source)" ]]; then mv $source/* $dest/ fi # maybe stop iterations on test failure - if [[ ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then break fi done - - (exit ${exit_code}) - fi - fi + done + (exit ${exit_code}) - store_test_results: - path: /tmp/results/repeated_utest/output + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/results/repeated_utest/stdout + path: /tmp/results/repeated_utests/stdout destination: stdout - store_artifacts: - path: /tmp/results/repeated_utest/output + path: /tmp/results/repeated_utests/output destination: junitxml - store_artifacts: - path: /tmp/results/repeated_utest/logs + path: /tmp/results/repeated_utests/logs destination: logs environment: - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 @@ -282,25 +280,370 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - repeated_upgrade_dtest: + utests_long_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_unit_tests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant testsome $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_upgrade_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -329,25 +672,25 @@ jobs: name: Run repeated Python dtest no_output_timeout: 15m command: | - if [ "${REPEATED_UPGRADE_DTEST_NAME}" == "" ]; then + if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" == "" ]; then + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" -le 0 ]; then + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" else # Calculate the number of test iterations to be run by the current parallel runner. # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then count=$((count+1)) fi if (($count <= 0)); then echo "No tests to run in this runner" else - echo "Running ${REPEATED_UPGRADE_DTEST_NAME} $count times" + echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" source ~/env3.6/bin/activate export PATH=$JAVA_HOME/bin:$PATH @@ -360,8 +703,10 @@ jobs: echo "** done env" mkdir -p /tmp/results/dtests + tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") + stop_on_failure_arg="" - if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then stop_on_failure_arg="-x" fi @@ -372,11 +717,11 @@ jobs: upgrade_arg="" if true; then - upgrade_arg="--execute-upgrade-tests" + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" fi # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} | tee /tmp/dtest/stdout.txt + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt fi fi - store_test_results: @@ -402,142 +747,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_repeated_dtest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Run repeated Python dtest - no_output_timeout: 15m - command: | - if [ "${REPEATED_DTEST_NAME}" == "" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" == "" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_DTEST_NAME} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - stop_on_failure_arg="" - if ${REPEATED_DTEST_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if ${REPEATED_DTEST_VNODES}; then - vnodes_args="--use-vnodes --num-tokens=16" - fi - - upgrade_arg="" - if false; then - upgrade_arg="--execute-upgrade-tests" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: @@ -619,25 +845,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-no-vnodes: + j8_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -663,21 +890,69 @@ jobs: pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt pip3 freeze - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - - run: - name: Run dtests (j8_without_vnodes) + name: Run repeated Python dtest no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: | + if [ "${REPEATED_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if true; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi - store_test_results: path: /tmp/results - store_artifacts: path: /tmp/dtest - destination: dtest_j8_without_vnodes + destination: dtest - store_artifacts: path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + destination: dtest_logs environment: - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - ANT_HOME: /usr/share/ant @@ -693,25 +968,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgradetests-no-vnodes: + j8_upgrade_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -808,22 +1084,195 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_unit_tests: @@ -916,25 +1365,101 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-with-vnodes: + j8_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -990,22 +1515,494 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated_ant_test: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run repeated JUnit test + no_output_timeout: 15m + command: | + if [ "${REPEATED_ANT_TEST_CLASS}" == "" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_ANT_TEST_COUNT}" == "" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=${REPEATED_ANT_TEST_TARGET} + class_path=${REPEATED_ANT_TEST_CLASS} + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "${REPEATED_ANT_TEST_METHODS}" == "" ]; then + methods="" + else + methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if stop_on_failure is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Run repeated Python dtest + no_output_timeout: 15m + command: | + if [ "${REPEATED_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_jvm_dtests: @@ -1098,22 +2095,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_long: @@ -1160,22 +2158,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_compression: @@ -1268,196 +2267,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_repeated_utest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run repeated JUnit test - no_output_timeout: 15m - command: | - if [ "${REPEATED_UTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times" - - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - target=${REPEATED_UTEST_TARGET} - class_path=${REPEATED_UTEST_CLASS} - class_name="${class_path##*.}" - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" - else - name="-Dtest.name=$class_path" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_UTEST_METHODS}" == "" ]; then - methods="" - else - methods="-Dtest.methods=${REPEATED_UTEST_METHODS}" - fi - - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" - for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt - - # move the XML output files - source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - - (exit ${exit_code}) - fi - fi - - store_test_results: - path: /tmp/results/repeated_utest/output - - store_artifacts: - path: /tmp/results/repeated_utest/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utest/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utest/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtest_jars_build: @@ -1536,22 +2362,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: @@ -1569,76 +2396,108 @@ workflows: requires: - start_j8_unit_tests - build + - start_j8_unit_tests_repeat: + type: approval + - j8_unit_tests_repeat: + requires: + - start_j8_unit_tests_repeat + - build - start_j8_jvm_dtests: type: approval - j8_jvm_dtests: requires: - start_j8_jvm_dtests - build + - start_j8_jvm_dtests_repeat: + type: approval + - j8_jvm_dtests_repeat: + requires: + - start_j8_jvm_dtests_repeat + - build - start_utests_long: type: approval - utests_long: requires: - start_utests_long - build + - start_utests_long_repeat: + type: approval + - utests_long_repeat: + requires: + - start_utests_long_repeat + - build - start_utests_compression: type: approval - utests_compression: requires: - start_utests_compression - build + - start_utests_compression_repeat: + type: approval + - utests_compression_repeat: + requires: + - start_utests_compression_repeat + - build - start_j8_dtest_jars_build: type: approval - j8_dtest_jars_build: requires: - start_j8_dtest_jars_build - build - - start_jvm_upgrade_dtest: + - start_jvm_upgrade_dtests: type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest + - start_jvm_upgrade_dtests + - j8_dtest_jars_build + - start_jvm_upgrade_dtests_repeat: + type: approval + - j8_jvm_upgrade_dtests_repeat: + requires: + - start_jvm_upgrade_dtests_repeat - j8_dtest_jars_build - start_j8_dtests: type: approval - - j8_dtests-with-vnodes: + - j8_dtests: requires: - start_j8_dtests - build - - j8_dtests-no-vnodes: - requires: - - start_j8_dtests - - build - - start_upgrade_tests: + - start_j8_dtests_repeat: type: approval - - j8_upgradetests-no-vnodes: + - j8_dtests_repeat: requires: - - start_upgrade_tests + - start_j8_dtests_repeat - build - - start_j8_repeated_utest: + - start_j8_dtests_vnode: type: approval - - j8_repeated_utest: + - j8_dtests_vnode: requires: - - start_j8_repeated_utest + - start_j8_dtests_vnode - build - - start_j8_repeated_dtest: + - start_j8_dtests_vnode_repeat: type: approval - - j8_repeated_dtest: + - j8_dtests_vnode_repeat: requires: - - start_j8_repeated_dtest + - start_j8_dtests_vnode_repeat - build - - start_repeated_upgrade_dtest: + - start_j8_upgrade_dtests: type: approval - - repeated_upgrade_dtest: + - j8_upgrade_dtests: requires: - - start_repeated_upgrade_dtest + - start_j8_upgrade_dtests - build - - start_repeated_jvm_upgrade_dtest: + - start_j8_upgrade_dtests_repeat: type: approval - - repeated_jvm_upgrade_dtest: + - j8_upgrade_dtests_repeat: requires: - - start_repeated_jvm_upgrade_dtest - - j8_dtest_jars_build + - start_j8_upgrade_dtests_repeat + - build + - start_j8_repeated_ant_test: + type: approval + - j8_repeated_ant_test: + requires: + - start_j8_repeated_ant_test + - build pre-commit_tests: jobs: - start_pre-commit_tests: @@ -1649,66 +2508,66 @@ workflows: - j8_unit_tests: requires: - build + - j8_unit_tests_repeat: + requires: + - build - j8_jvm_dtests: requires: - build + - j8_jvm_dtests_repeat: + requires: + - build - start_utests_long: type: approval - utests_long: requires: - start_utests_long - build + - utests_long_repeat: + requires: + - start_utests_long + - build - start_utests_compression: type: approval - utests_compression: requires: - start_utests_compression - build - - start_j8_dtest_jars_build: + - utests_compression_repeat: + requires: + - start_utests_compression + - build + - start_jvm_upgrade_dtests: type: approval - j8_dtest_jars_build: requires: - - start_j8_dtest_jars_build + - start_jvm_upgrade_dtests - build - - start_jvm_upgrade_dtest: - type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest - j8_dtest_jars_build - - j8_dtests-with-vnodes: + - j8_jvm_upgrade_dtests_repeat: requires: - - build - - j8_dtests-no-vnodes: - requires: - - build - - start_upgrade_tests: - type: approval - - j8_upgradetests-no-vnodes: - requires: - - start_upgrade_tests - - build - - start_j8_repeated_utest: - type: approval - - j8_repeated_utest: - requires: - - start_j8_repeated_utest - - build - - start_j8_repeated_dtest: - type: approval - - j8_repeated_dtest: - requires: - - start_j8_repeated_dtest - - build - - start_repeated_upgrade_dtest: - type: approval - - repeated_upgrade_dtest: - requires: - - start_repeated_upgrade_dtest - - build - - start_repeated_jvm_upgrade_dtest: - type: approval - - repeated_jvm_upgrade_dtest: - requires: - - start_repeated_jvm_upgrade_dtest - j8_dtest_jars_build + - j8_dtests: + requires: + - build + - j8_dtests_repeat: + requires: + - build + - j8_dtests_vnode: + requires: + - build + - j8_dtests_vnode_repeat: + requires: + - build + - start_upgrade_dtests: + type: approval + - j8_upgrade_dtests: + requires: + - start_upgrade_dtests + - build + - j8_upgrade_dtests_repeat: + requires: + - start_upgrade_dtests + - build diff --git a/.circleci/config.yml.MIDRES b/.circleci/config.yml.MIDRES index d13d6643f9..38a2fa39ef 100644 --- a/.circleci/config.yml.MIDRES +++ b/.circleci/config.yml.MIDRES @@ -108,25 +108,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - repeated_jvm_upgrade_dtest: + utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: medium @@ -159,81 +160,80 @@ jobs: which java java -version - run: - name: Run repeated JUnit test + name: Repeatedly run new or modifed JUnit tests no_output_timeout: 15m command: | - if [ "${REPEATED_JVM_UPGRADE_DTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_JVM_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running test-jvm-dtest-some ${REPEATED_JVM_UPGRADE_DTEST_CLASS} ${REPEATED_JVM_UPGRADE_DTEST_METHODS} ${REPEATED_JVM_UPGRADE_DTEST_COUNT} times" + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" fi - target=test-jvm-dtest-some - class_path=${REPEATED_JVM_UPGRADE_DTEST_CLASS} - class_name="${class_path##*.}" - # Prepare the -Dtest.name argument. # It can be the fully qualified class name or the short class name, depending on the target. if [[ $target == "test" || \ $target == "test-cdc" || \ $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" else - name="-Dtest.name=$class_path" + name_arg="-Dtest.name=$class" fi # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_JVM_UPGRADE_DTEST_METHODS}" == "" ]; then - methods="" + if [ $method == "" ]; then + methods_arg="" else - methods="-Dtest.methods=${REPEATED_JVM_UPGRADE_DTEST_METHODS}" + methods_arg="-Dtest.methods=$method" fi - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" + echo "Running test $test, iteration $i of $count" # run the test status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + if !( set -o pipefail && \ + ant test-compression $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then status="fails" exit_code=1 fi # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} + dest=/tmp/results/repeated_utests/stdout/${status}/${i} mkdir -p $dest - mv stdout.txt $dest/test-jvm-dtest-some-${REPEATED_JVM_UPGRADE_DTEST_CLASS}.txt + mv stdout.txt $dest/${test}.txt # move the XML output files source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} + dest=/tmp/results/repeated_utests/output/${status}/${i} mkdir -p $dest if [[ -d $source && -n "$(ls $source)" ]]; then mv $source/* $dest/ @@ -241,31 +241,29 @@ jobs: # move the log files source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} + dest=/tmp/results/repeated_utests/logs/${status}/${i} mkdir -p $dest if [[ -d $source && -n "$(ls $source)" ]]; then mv $source/* $dest/ fi # maybe stop iterations on test failure - if [[ ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then break fi done - - (exit ${exit_code}) - fi - fi + done + (exit ${exit_code}) - store_test_results: - path: /tmp/results/repeated_utest/output + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/results/repeated_utest/stdout + path: /tmp/results/repeated_utests/stdout destination: stdout - store_artifacts: - path: /tmp/results/repeated_utest/output + path: /tmp/results/repeated_utests/output destination: junitxml - store_artifacts: - path: /tmp/results/repeated_utest/logs + path: /tmp/results/repeated_utests/logs destination: logs environment: - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 @@ -282,25 +280,370 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - repeated_upgrade_dtest: + utests_long_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_unit_tests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_UTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant testsome $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_upgrade_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: xlarge @@ -329,25 +672,25 @@ jobs: name: Run repeated Python dtest no_output_timeout: 15m command: | - if [ "${REPEATED_UPGRADE_DTEST_NAME}" == "" ]; then + if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" == "" ]; then + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" -le 0 ]; then + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" else # Calculate the number of test iterations to be run by the current parallel runner. # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then count=$((count+1)) fi if (($count <= 0)); then echo "No tests to run in this runner" else - echo "Running ${REPEATED_UPGRADE_DTEST_NAME} $count times" + echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" source ~/env3.6/bin/activate export PATH=$JAVA_HOME/bin:$PATH @@ -360,8 +703,10 @@ jobs: echo "** done env" mkdir -p /tmp/results/dtests + tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") + stop_on_failure_arg="" - if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then stop_on_failure_arg="-x" fi @@ -372,11 +717,11 @@ jobs: upgrade_arg="" if true; then - upgrade_arg="--execute-upgrade-tests" + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" fi # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} | tee /tmp/dtest/stdout.txt + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt fi fi - store_test_results: @@ -402,142 +747,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_repeated_dtest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: large - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 25 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Run repeated Python dtest - no_output_timeout: 15m - command: | - if [ "${REPEATED_DTEST_NAME}" == "" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" == "" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_DTEST_NAME} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - stop_on_failure_arg="" - if ${REPEATED_DTEST_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if ${REPEATED_DTEST_VNODES}; then - vnodes_args="--use-vnodes --num-tokens=16" - fi - - upgrade_arg="" - if false; then - upgrade_arg="--execute-upgrade-tests" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: @@ -619,31 +845,32 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-no-vnodes: + j8_dtests_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: large working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 50 + parallelism: 25 steps: - attach_workspace: at: /home/cassandra @@ -663,21 +890,69 @@ jobs: pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt pip3 freeze - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - - run: - name: Run dtests (j8_without_vnodes) + name: Run repeated Python dtest no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: | + if [ "${REPEATED_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if true; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi - store_test_results: path: /tmp/results - store_artifacts: path: /tmp/dtest - destination: dtest_j8_without_vnodes + destination: dtest - store_artifacts: path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + destination: dtest_logs environment: - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - ANT_HOME: /usr/share/ant @@ -693,25 +968,26 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgradetests-no-vnodes: + j8_upgrade_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: xlarge @@ -808,22 +1084,195 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_unit_tests: @@ -916,25 +1365,101 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-with-vnodes: + j8_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 50 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest resource_class: large @@ -990,22 +1515,494 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + # Calculate the number of test iterations to be run by the current parallel runner. + count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + # Put manually specified tests and automatically detected tests together, removing duplicates + tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s///" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) + echo "Tests to be repeated: ${tests}" + + # Run each test class as many times as requested. + exit_code="$?" + for test in $tests; do + + # Split class and method names from the test name + if [[ $test =~ "#" ]]; then + class=${test%"#"*} + method=${test#*"#"} + else + class=$test + method="" + fi + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name_arg="-Dtest.name=${class##*.}" + else + name_arg="-Dtest.name=$class" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ $method == "" ]; then + methods_arg="" + else + methods_arg="-Dtest.methods=$method" + fi + + for i in $(seq -w 1 $count); do + echo "Running test $test, iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && \ + ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \ + tee stdout.txt \ + ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utests/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${test}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utests/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utests/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + done + (exit ${exit_code}) + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_repeated_ant_test: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run repeated JUnit test + no_output_timeout: 15m + command: | + if [ "${REPEATED_ANT_TEST_CLASS}" == "" ]; then + echo "Repeated utest class name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_ANT_TEST_COUNT}" == "" ]; then + echo "Repeated utest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then + echo "Repeated utest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times" + + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + + target=${REPEATED_ANT_TEST_TARGET} + class_path=${REPEATED_ANT_TEST_CLASS} + class_name="${class_path##*.}" + + # Prepare the -Dtest.name argument. + # It can be the fully qualified class name or the short class name, depending on the target. + if [[ $target == "test" || \ + $target == "test-cdc" || \ + $target == "test-compression" || \ + $target == "test-system-keyspace-directory" || \ + $target == "long-test" ]]; then + name="-Dtest.name=$class_name" + else + name="-Dtest.name=$class_path" + fi + + # Prepare the -Dtest.methods argument, which is optional + if [ "${REPEATED_ANT_TEST_METHODS}" == "" ]; then + methods="" + else + methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}" + fi + + # Run the test target as many times as requested collecting the exit code, + # stopping the iteration only if stop_on_failure is set. + exit_code="$?" + for i in $(seq -w 1 $count); do + + echo "Running test iteration $i of $count" + + # run the test + status="passes" + if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then + status="fails" + exit_code=1 + fi + + # move the stdout output file + dest=/tmp/results/repeated_utest/stdout/${status}/${i} + mkdir -p $dest + mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt + + # move the XML output files + source=build/test/output + dest=/tmp/results/repeated_utest/output/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # move the log files + source=build/test/logs + dest=/tmp/results/repeated_utest/logs/${status}/${i} + mkdir -p $dest + if [[ -d $source && -n "$(ls $source)" ]]; then + mv $source/* $dest/ + fi + + # maybe stop iterations on test failure + if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then + break + fi + done + + (exit ${exit_code}) + fi + fi + - store_test_results: + path: /tmp/results/repeated_utest/output + - store_artifacts: + path: /tmp/results/repeated_utest/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utest/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utest/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Run repeated Python dtest + no_output_timeout: 15m + command: | + if [ "${REPEATED_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if false; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_jvm_dtests: @@ -1098,22 +2095,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_long: @@ -1160,22 +2158,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_compression: @@ -1268,196 +2267,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_repeated_utest: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 25 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run repeated JUnit test - no_output_timeout: 15m - command: | - if [ "${REPEATED_UTEST_CLASS}" == "" ]; then - echo "Repeated utest class name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" == "" ]; then - echo "Repeated utest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then - echo "Repeated utest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times" - - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - - target=${REPEATED_UTEST_TARGET} - class_path=${REPEATED_UTEST_CLASS} - class_name="${class_path##*.}" - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" ]]; then - name="-Dtest.name=$class_name" - else - name="-Dtest.name=$class_path" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [ "${REPEATED_UTEST_METHODS}" == "" ]; then - methods="" - else - methods="-Dtest.methods=${REPEATED_UTEST_METHODS}" - fi - - # Run the test target as many times as requested collecting the exit code, - # stopping the iteration only if stop_on_failure is set. - exit_code="$?" - for i in $(seq -w 1 $count); do - - echo "Running test iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utest/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt - - # move the XML output files - source=build/test/output - dest=/tmp/results/repeated_utest/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs - dest=/tmp/results/repeated_utest/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - - (exit ${exit_code}) - fi - fi - - store_test_results: - path: /tmp/results/repeated_utest/output - - store_artifacts: - path: /tmp/results/repeated_utest/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utest/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utest/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtest_jars_build: @@ -1536,22 +2362,23 @@ jobs: - DTEST_BRANCH: trunk - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - - REPEATED_UTEST_TARGET: testsome - - REPEATED_UTEST_CLASS: null - - REPEATED_UTEST_METHODS: null - - REPEATED_UTEST_COUNT: 100 - - REPEATED_UTEST_STOP_ON_FAILURE: false - - REPEATED_DTEST_NAME: null - - REPEATED_DTEST_VNODES: false - - REPEATED_DTEST_COUNT: 100 - - REPEATED_DTEST_STOP_ON_FAILURE: false - - REPEATED_UPGRADE_DTEST_NAME: null - - REPEATED_UPGRADE_DTEST_COUNT: 100 - - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false - - REPEATED_JVM_UPGRADE_DTEST_CLASS: null - - REPEATED_JVM_UPGRADE_DTEST_METHODS: null - - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100 - - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: @@ -1569,76 +2396,108 @@ workflows: requires: - start_j8_unit_tests - build + - start_j8_unit_tests_repeat: + type: approval + - j8_unit_tests_repeat: + requires: + - start_j8_unit_tests_repeat + - build - start_j8_jvm_dtests: type: approval - j8_jvm_dtests: requires: - start_j8_jvm_dtests - build + - start_j8_jvm_dtests_repeat: + type: approval + - j8_jvm_dtests_repeat: + requires: + - start_j8_jvm_dtests_repeat + - build - start_utests_long: type: approval - utests_long: requires: - start_utests_long - build + - start_utests_long_repeat: + type: approval + - utests_long_repeat: + requires: + - start_utests_long_repeat + - build - start_utests_compression: type: approval - utests_compression: requires: - start_utests_compression - build + - start_utests_compression_repeat: + type: approval + - utests_compression_repeat: + requires: + - start_utests_compression_repeat + - build - start_j8_dtest_jars_build: type: approval - j8_dtest_jars_build: requires: - start_j8_dtest_jars_build - build - - start_jvm_upgrade_dtest: + - start_jvm_upgrade_dtests: type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest + - start_jvm_upgrade_dtests + - j8_dtest_jars_build + - start_jvm_upgrade_dtests_repeat: + type: approval + - j8_jvm_upgrade_dtests_repeat: + requires: + - start_jvm_upgrade_dtests_repeat - j8_dtest_jars_build - start_j8_dtests: type: approval - - j8_dtests-with-vnodes: + - j8_dtests: requires: - start_j8_dtests - build - - j8_dtests-no-vnodes: - requires: - - start_j8_dtests - - build - - start_upgrade_tests: + - start_j8_dtests_repeat: type: approval - - j8_upgradetests-no-vnodes: + - j8_dtests_repeat: requires: - - start_upgrade_tests + - start_j8_dtests_repeat - build - - start_j8_repeated_utest: + - start_j8_dtests_vnode: type: approval - - j8_repeated_utest: + - j8_dtests_vnode: requires: - - start_j8_repeated_utest + - start_j8_dtests_vnode - build - - start_j8_repeated_dtest: + - start_j8_dtests_vnode_repeat: type: approval - - j8_repeated_dtest: + - j8_dtests_vnode_repeat: requires: - - start_j8_repeated_dtest + - start_j8_dtests_vnode_repeat - build - - start_repeated_upgrade_dtest: + - start_j8_upgrade_dtests: type: approval - - repeated_upgrade_dtest: + - j8_upgrade_dtests: requires: - - start_repeated_upgrade_dtest + - start_j8_upgrade_dtests - build - - start_repeated_jvm_upgrade_dtest: + - start_j8_upgrade_dtests_repeat: type: approval - - repeated_jvm_upgrade_dtest: + - j8_upgrade_dtests_repeat: requires: - - start_repeated_jvm_upgrade_dtest - - j8_dtest_jars_build + - start_j8_upgrade_dtests_repeat + - build + - start_j8_repeated_ant_test: + type: approval + - j8_repeated_ant_test: + requires: + - start_j8_repeated_ant_test + - build pre-commit_tests: jobs: - start_pre-commit_tests: @@ -1649,66 +2508,66 @@ workflows: - j8_unit_tests: requires: - build + - j8_unit_tests_repeat: + requires: + - build - j8_jvm_dtests: requires: - build + - j8_jvm_dtests_repeat: + requires: + - build - start_utests_long: type: approval - utests_long: requires: - start_utests_long - build + - utests_long_repeat: + requires: + - start_utests_long + - build - start_utests_compression: type: approval - utests_compression: requires: - start_utests_compression - build - - start_j8_dtest_jars_build: + - utests_compression_repeat: + requires: + - start_utests_compression + - build + - start_jvm_upgrade_dtests: type: approval - j8_dtest_jars_build: requires: - - start_j8_dtest_jars_build + - start_jvm_upgrade_dtests - build - - start_jvm_upgrade_dtest: - type: approval - j8_jvm_upgrade_dtests: requires: - - start_jvm_upgrade_dtest - j8_dtest_jars_build - - j8_dtests-with-vnodes: + - j8_jvm_upgrade_dtests_repeat: requires: - - build - - j8_dtests-no-vnodes: - requires: - - build - - start_upgrade_tests: - type: approval - - j8_upgradetests-no-vnodes: - requires: - - start_upgrade_tests - - build - - start_j8_repeated_utest: - type: approval - - j8_repeated_utest: - requires: - - start_j8_repeated_utest - - build - - start_j8_repeated_dtest: - type: approval - - j8_repeated_dtest: - requires: - - start_j8_repeated_dtest - - build - - start_repeated_upgrade_dtest: - type: approval - - repeated_upgrade_dtest: - requires: - - start_repeated_upgrade_dtest - - build - - start_repeated_jvm_upgrade_dtest: - type: approval - - repeated_jvm_upgrade_dtest: - requires: - - start_repeated_jvm_upgrade_dtest - j8_dtest_jars_build + - j8_dtests: + requires: + - build + - j8_dtests_repeat: + requires: + - build + - j8_dtests_vnode: + requires: + - build + - j8_dtests_vnode_repeat: + requires: + - build + - start_upgrade_dtests: + type: approval + - j8_upgrade_dtests: + requires: + - start_upgrade_dtests + - build + - j8_upgrade_dtests_repeat: + requires: + - start_upgrade_dtests + - build diff --git a/.circleci/generate.sh b/.circleci/generate.sh index 4f9430572a..fa109564e4 100755 --- a/.circleci/generate.sh +++ b/.circleci/generate.sh @@ -18,6 +18,7 @@ # BASEDIR=`dirname $0` +BASE_BRANCH=cassandra-3.0 die () { @@ -38,15 +39,23 @@ print_help() echo " -e Environment variables to be used in the generated config.yml, e.g.:" echo " -e DTEST_BRANCH=CASSANDRA-8272" echo " -e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git" - echo " -e REPEATED_UTEST_TARGET=testsome" - echo " -e REPEATED_UTEST_CLASS=org.apache.cassandra.cql3.ViewTest" - echo " -e REPEATED_UTEST_METHODS=testCompoundPartitionKey,testStaticTable" - echo " -e REPEATED_UTEST_COUNT=100" - echo " -e REPEATED_UTEST_STOP_ON_FAILURE=false" - echo " -e REPEATED_DTEST_NAME=cqlsh_tests/test_cqlsh.py::TestCqlshSmoke" - echo " -e REPEATED_DTEST_VNODES=false" - echo " -e REPEATED_DTEST_COUNT=100" - echo " -e REPEATED_DTEST_STOP_ON_FAILURE=false" + echo " -e REPEATED_TESTS_STOP_ON_FAILURE=false" + echo " -e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest#testCountersTable" + echo " -e REPEATED_UTESTS_COUNT=500" + echo " -e REPEATED_UTESTS_LONG=org.apache.cassandra.db.commitlog.CommitLogStressTest" + echo " -e REPEATED_UTESTS_LONG_COUNT=100" + echo " -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest" + echo " -e REPEATED_JVM_DTESTS_COUNT=500" + echo " -e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest" + echo " -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500" + echo " -e REPEATED_DTESTS=cdc_test.py cqlsh_tests/test_cqlsh.py::TestCqlshSmoke" + echo " -e REPEATED_DTESTS_COUNT=500" + echo " -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py upgrade_tests/paging_test.py" + echo " -e REPEATED_UPGRADE_DTESTS_COUNT=25" + echo " -e REPEATED_ANT_TEST_TARGET=testsome" + echo " -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest" + echo " -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable" + echo " -e REPEATED_ANT_TEST_COUNT=500" echo " For the complete list of environment variables, please check the" echo " list of examples in config-2_1.yml and/or the documentation." echo " If you want to specify multiple environment variables simply add" @@ -95,22 +104,23 @@ if $has_env_vars && $check_env_vars; then key=$(echo $entry | tr "=" "\n" | head -n 1) if [ "$key" != "DTEST_REPO" ] && [ "$key" != "DTEST_BRANCH" ] && - [ "$key" != "REPEATED_UTEST_TARGET" ] && - [ "$key" != "REPEATED_UTEST_CLASS" ] && - [ "$key" != "REPEATED_UTEST_METHODS" ] && - [ "$key" != "REPEATED_UTEST_COUNT" ] && - [ "$key" != "REPEATED_UTEST_STOP_ON_FAILURE" ] && - [ "$key" != "REPEATED_DTEST_NAME" ] && - [ "$key" != "REPEATED_DTEST_VNODES" ] && - [ "$key" != "REPEATED_DTEST_COUNT" ] && - [ "$key" != "REPEATED_DTEST_STOP_ON_FAILURE" ] && - [ "$key" != "REPEATED_UPGRADE_DTEST_NAME" ] && - [ "$key" != "REPEATED_UPGRADE_DTEST_COUNT" ] && - [ "$key" != "REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE" ] && - [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_CLASS" ] && - [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_METHODS" ] && - [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_COUNT" ] && - [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE" ]; then + [ "$key" != "REPEATED_TESTS_STOP_ON_FAILURE" ] && + [ "$key" != "REPEATED_UTESTS" ] && + [ "$key" != "REPEATED_UTESTS_COUNT" ] && + [ "$key" != "REPEATED_UTESTS_LONG" ] && + [ "$key" != "REPEATED_UTESTS_LONG_COUNT" ] && + [ "$key" != "REPEATED_JVM_DTESTS" ] && + [ "$key" != "REPEATED_JVM_DTESTS_COUNT" ] && + [ "$key" != "REPEATED_JVM_UPGRADE_DTESTS" ] && + [ "$key" != "REPEATED_JVM_UPGRADE_DTESTS_COUNT" ] && + [ "$key" != "REPEATED_DTESTS" ] && + [ "$key" != "REPEATED_DTESTS_COUNT" ] && + [ "$key" != "REPEATED_UPGRADE_DTESTS" ] && + [ "$key" != "REPEATED_UPGRADE_DTESTS_COUNT" ] && + [ "$key" != "REPEATED_ANT_TEST_TARGET" ] && + [ "$key" != "REPEATED_ANT_TEST_CLASS" ] && + [ "$key" != "REPEATED_ANT_TEST_METHODS" ] && + [ "$key" != "REPEATED_ANT_TEST_COUNT" ]; then die "Unrecognised environment variable name: $key" fi done @@ -167,17 +177,96 @@ elif (!($has_env_vars)); then print_help fi +# add new or modified tests to the sets of tests to be repeated +if (!($all)); then + add_diff_tests () + { + dir="${BASEDIR}/../${2}" + diff=$(git --no-pager diff --name-only --diff-filter=AMR ${BASE_BRANCH}...HEAD ${dir}) + tests=$( echo "$diff" \ + | grep "Test\\.java" \ + | sed -e "s/\\.java//" \ + | sed -e "s,^${2},," \ + | tr '/' '.' \ + | grep ${3} ) + for test in $tests; do + echo " $test" + has_env_vars=true + if grep -q "${1}=" <<< "$env_vars"; then + env_vars=$(sed -e "s/${1}=/${1}=${test},/" <<< $env_vars) + elif [[ $env_vars == "" ]]; then + env_vars="${1}=${test}" + else + env_vars="$env_vars|${1}=${test}" + fi + done + } + + echo + echo "Detecting new or modified tests with git diff --diff-filter=AMR ${BASE_BRANCH}...HEAD:" + add_diff_tests "REPEATED_UTESTS" "test/unit/" "org.apache.cassandra" + add_diff_tests "REPEATED_UTESTS_LONG" "test/long/" "org.apache.cassandra" + add_diff_tests "REPEATED_JVM_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.test" + add_diff_tests "REPEATED_JVM_UPGRADE_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.upgrade" +fi + # replace environment variables if $has_env_vars; then + echo + echo "Setting environment variables:" IFS='=' echo "$env_vars" | tr '|' '\n' | while read entry; do set -- $entry key=$1 val=$2 - echo "Setting environment variable $key: $val" + echo " $key: $val" sed -i.bak "s|- $key:.*|- $key: $val|" $BASEDIR/config.yml done unset IFS fi +# define function to remove unneeded jobs +delete_job() +{ + delete_yaml_block() + { + sed -Ei.bak "/^ - ${1}/,/^ [^[:space:]]+|^ [^[:space:]]+/{//!d;}" $BASEDIR/config.yml + sed -Ei.bak "/^ - ${1}/d" $BASEDIR/config.yml + } + delete_yaml_block "${1}" + delete_yaml_block "start_${1}" +} +# removed unneeded repeated jobs +if [[ $env_vars != *"REPEATED_UTESTS="* ]]; then + delete_job "j8_unit_tests_repeat" + delete_job "j11_unit_tests_repeat" + delete_job "utests_compression_repeat" + delete_job "utests_system_keyspace_directory_repeat" +fi +if [[ $env_vars != *"REPEATED_UTESTS_LONG="* ]]; then + delete_job "utests_long_repeat" +fi +if [[ $env_vars != *"REPEATED_JVM_DTESTS="* ]]; then + delete_job "j8_jvm_dtests_repeat" + delete_job "j8_jvm_dtests_vnode_repeat" + delete_job "j11_jvm_dtests_repeat" + delete_job "j11_jvm_dtests_vnode_repeat" +fi +if [[ $env_vars != *"REPEATED_JVM_UPGRADE_DTESTS="* ]]; then + delete_job "start_jvm_upgrade_dtests_repeat" + delete_job "j8_jvm_upgrade_dtests_repeat" +fi +if [[ $env_vars != *"REPEATED_DTESTS="* ]]; then + delete_job "j8_dtests_repeat" + delete_job "j8_dtests_vnode_repeat" + delete_job "j11_dtests_repeat" + delete_job "j11_dtests_vnode_repeat" +fi +if [[ $env_vars != *"REPEATED_UPGRADE_DTESTS="* ]]; then + delete_job "j8_upgrade_dtests_repeat" +fi +if [[ $env_vars != *"REPEATED_ANT_TEST_CLASS="* ]]; then + delete_job "j8_repeated_ant_test" + delete_job "j11_repeated_ant_test" +fi diff --git a/.circleci/readme.md b/.circleci/readme.md index ab7d5a9483..ddc5076354 100644 --- a/.circleci/readme.md +++ b/.circleci/readme.md @@ -22,7 +22,7 @@ This directory contains the configuration for CircleCI continous integration platform. The file `config.yml` is the configuration file that is read by CircleCI. This file is -automatically generated by the `generate.sh` script from the `config-2_1.yml` file. +automatically generated by the `generate.sh` script from the `config-2_1.yml` file. The provided `config.yml` file uses low resources so users of the CircleCI free tier can use it. Additionally, there are three versions of this file using different resources so @@ -43,33 +43,86 @@ requires the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to installed. ## Setting environment variables -Both `config-2_1.yml` and `config.yml` files contain a set of environment variables -defining things like what dtest repo and branch to use, what tests could be repeatedly +Both `config-2_1.yml` and `config.yml` files contain a set of environment variables +defining things like what dtest repo and branch to use, what tests could be repeatedly run, etc. -These environment variables can be directly edited in the `config.yml` file, although if -you do this you should take into account that the entire set of env vars is repeated on +These environment variables can be directly edited in the `config.yml` file, although if +you do this you should take into account that the entire set of env vars is repeated on every job. -A probably better approach is editing them in `config-2_1.yml` and then regenerate the +A probably better approach is editing them in `config-2_1.yml` and then regenerate the `config.yml` file using the `generate.sh` script. You can also directly pass environment -variable values to the `generate.sh` script with the `-e` flag. For example, to set the +variable values to the `generate.sh` script with the `-e` flag. For example, to set the dtest repo and branch with MIDRES config you can run: ``` generate.sh -m \ -e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git \ - -e DTEST_BRANCH=CASSANDRA-8272 -``` - -Or you can set the test multiplexer for repeating a specific test with HIGHRES: + -e DTEST_BRANCH=CASSANDRA-8272 ``` -generate.sh -h \ - -e REPEATED_UTEST_TARGET=testsome \ - -e REPEATED_UTEST_CLASS=org.apache.cassandra.cql3.ViewTest \ - -e REPEATED_UTEST_METHODS=testCompoundPartitionKey,testStaticTable \ - -e REPEATED_UTEST_COUNT=100 + +## Running tests in a loop +Running the `generate.sh` script with use `git diff` to find the new or modified tests. +The script will then create jobs to run each of these new or modified tests for a certain +number of times, to verify that they are stable. You can use environment variables to +specify the number of iterations of each type of test: +``` +generate.sh -m \ + -e REPEATED_UTESTS_COUNT=500 \ + -e REPEATED_UTESTS_LONG_COUNT=100 \ + -e REPEATED_SIMULATOR_DTESTS_COUNT=500 \ + -e REPEATED_JVM_DTESTS_COUNT=500 \ + -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500 \ + -e REPEATED_DTESTS_COUNT=500 \ + -e REPEATED_UPGRADE_DTESTS_COUNT=25 \ + -e REPEATED_ANT_TEST_COUNT=500 +``` +You can also specify whether the iteration should fail on the first test failure: +``` +generate.sh -m -e REPEATED_TESTS_STOP_ON_FAILURE=false +``` +In addition to the automatically detected tests, it's also possible to provide lists of +specific tests to be repeated: +``` +generate.sh -m \ + -e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest \ + -e REPEATED_UTESTS_LONG=org.apache.cassandra.io.sstable.CQLSSTableWriterLongTest#testWideRow \ + -e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \ + -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest#testPaging \ + -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py \ + -e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest +``` +For particular Ant test targets that are not included in the regular test suites, you can +use the `run_repeated_utest` job: +``` +generate.sh -m \ + -e REPEATED_ANT_TEST_TARGET=test-cdc \ + -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest \ + -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable \ + -e REPEATED_ANT_TEST_COUNT=500 +``` +Putting all together, you can have runs as complex as: +``` +generate.sh -m \ + -e REPEATED_TESTS_STOP_ON_FAILURE=true \ + -e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest \ + -e REPEATED_UTESTS_COUNT=500 \ + -e REPEATED_UTESTS_LONG=org.apache.cassandra.io.sstable.CQLSSTableWriterLongTest#testWideRow \ + -e REPEATED_UTESTS_LONG_COUNT=100 \ + -e REPEATED_DTESTS=cql_test.py,consistency_test.py::TestAvailability::test_simple_strategy \ + -e REPEATED_DTESTS_COUNT=500 \ + -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest#testPaging \ + -e REPEATED_JVM_DTESTS_COUNT=500 \ + -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py \ + -e REPEATED_UPGRADE_DTESTS_COUNT=25 \ + -e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest \ + -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500 \ + -e REPEATED_ANT_TEST_TARGET=test-cdc \ + -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest \ + -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable \ + -e REPEATED_ANT_TEST_COUNT=500 ``` ## Updating the config @@ -78,7 +131,7 @@ the `config.yml` file manually. Instead, you should edit the `config-2_1.yml` fi regenerate the `config.yml`, `config.yml.LOWRES`, `config.yml.MIDRES` and `config.yml.HIGHRES` files by runnining the `generate.sh` script without any flags. For using this script you need to install the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install). - + As for temporal changes done while working in a patch, such as pointing to you dtest repo or running a test repeatedly, you can either directly edit `config.yml` or edit `config-2_1.yml` and then regenerate `config.yml` with the `generate.sh` script using a `-l`/`-m`/`-h` flag. @@ -92,27 +145,27 @@ again or use the `-l`/`-m`/`-h` script flags. Read below for details how to generate the files manually without the `generate.sh` script: 1. make your edits to config-2_1.yml - let it stay at lowres settings -1. generate a valid LOWRES file: +2. generate a valid LOWRES file: `circleci config process config-2_1.yml > config.yml.LOWRES` -1. add the Apache license header to the newly created LOWRES file: +3. add the Apache license header to the newly created LOWRES file: `cat license.yml config.yml.LOWRES > config.yml.LOWRES.new && mv config.yml.LOWRES.new config.yml.LOWRES` -1. then apply the highres patch to config-2_1.yml; +4. then apply the highres patch to config-2_1.yml; `patch -o config-2_1.yml.HIGHRES config-2_1.yml config-2_1.yml.high_res.patch` (this creates a new file `config-2_1.yml.HIGHRES` instead of in-place patching config-2_1.yml) Note that if the patch no longer applies to `config-2_1.yml` a new patch file is needed, do this by manually making `config-2_1.yml` high resource and create the patch file based on the diff (don't commit it though). -1. generate the HIGHRES file: +5. generate the HIGHRES file: `circleci config process config-2_1.yml.HIGHRES > config.yml.HIGHRES` -1. remove the temporary patched HIGHRES file: `rm config-2_1.yml.HIGHRES` -1. add the Apache license header to the newly created HIGHRES file: +6. remove the temporary patched HIGHRES file: `rm config-2_1.yml.HIGHRES` +7. add the Apache license header to the newly created HIGHRES file: `cat license.yml config.yml.HIGHRES > config.yml.HIGHRES.new && mv config.yml.HIGHRES.new config.yml.HIGHRES` -1. repeat the last steps to generate the MIDRES file: +8. repeat the last steps to generate the MIDRES file: ``` patch -o config-2_1.yml.MIDRES config-2_1.yml config-2_1.yml.mid_res.patch circleci config process config-2_1.yml.MIDRES > config.yml.MIDRES rm config-2_1.yml.MIDRES cat license.yml config.yml.MIDRES > config.yml.MIDRES.new && mv config.yml.MIDRES.new config.yml.MIDRES ``` - +9. finally, remember to update the config.yml diff --git a/build.xml b/build.xml index db0eacef80..3b46c82bbe 100644 --- a/build.xml +++ b/build.xml @@ -531,7 +531,7 @@ - + @@ -1333,6 +1333,19 @@ + + + + + + + + + +