From 1d961679a0ab76043e610e45aa7089e81fd64aa1 Mon Sep 17 00:00:00 2001 From: Ekaterina Dimitrova Date: Mon, 5 Jun 2023 21:47:09 -0400 Subject: [PATCH] Switch Java and Python upgrade tests from running on JDK8 to JDK11 in CircleCI patch by Ekaterina Dimitrova; reviewed by Berenguer Blasi for CASSANDRA-18285 --- .circleci/config.yml | 969 ++++++++--------- .circleci/config.yml.FREE | 969 ++++++++--------- .circleci/config.yml.PAID | 971 ++++++++--------- .circleci/config_11_and_17.yml | 978 ++++++++++++++++++ .circleci/config_11_and_17.yml.FREE | 978 ++++++++++++++++++ .circleci/config_11_and_17.yml.PAID | 978 ++++++++++++++++++ .circleci/config_template.yml | 146 +-- .circleci/config_template.yml.PAID.patch | 86 +- .circleci/config_template_11_and_17.yml | 144 ++- .../config_template_11_and_17.yml.PAID.patch | 89 +- .circleci/generate.sh | 4 +- .circleci/generate_11_and_17.sh | 16 + 12 files changed, 4730 insertions(+), 1598 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ce806829c..df8682ec52 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,35 +18,16 @@ version: 2 jobs: - j8_jvm_upgrade_dtests: + j11_jvm_upgrade_dtests_repeat: docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + - image: apache/cassandra-testing-ubuntu2004-java11:latest resource_class: medium working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 1 + parallelism: 4 steps: - attach_workspace: at: /home/cassandra - - run: - name: Determine distributed Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - run: name: Log Environment Information command: | @@ -70,28 +51,19 @@ jobs: which java java -version - run: - name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + name: Repeatedly run new or modifed JUnit tests no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s///\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" - store_test_results: - path: /tmp/cassandra/build/test/output/ + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/cassandra/build/test/output + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output destination: junitxml - store_artifacts: - path: /tmp/cassandra/build/test/logs + path: /tmp/results/repeated_utests/logs destination: logs environment: - ANT_HOME: /usr/share/ant @@ -133,8 +105,9 @@ jobs: - REPEATED_ANT_TEST_METHODS: null - REPEATED_ANT_TEST_VNODES: false - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_fqltool_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2668,6 +2641,139 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Run repeated Python DTests + no_output_timeout: 15m + command: | + if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if true; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_cqlsh_dtests_py3_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4000,6 +4106,110 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_dtest_jars_build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + mkdir ~/dtest_jars + git remote add apache https://github.com/apache/cassandra.git + for branch in cassandra-4.0 cassandra-4.1 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + git clean -fd + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + ${ANT_HOME}/bin/ant realclean + git checkout origin/$CIRCLE_BRANCH + git clean -fd + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_long: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4359,6 +4569,124 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_jvm_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_stress: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4539,138 +4867,6 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgrade_dtests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Run repeated Python DTests - no_output_timeout: 15m - command: | - if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") - - stop_on_failure_arg="" - if ${REPEATED_TESTS_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if false; then - vnodes_args="--use-vnodes --num-tokens=16" - fi - - upgrade_arg="" - if true; then - upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_utests_cdc_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5951,6 +6147,91 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_upgradetests_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j11_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j11_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_upgradetests_without_vnodes > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_upgradetests_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_upgradetests_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_upgradetests_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6639,90 +6920,6 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_upgrade_dtests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j8_upgradetests_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n" - - run: - name: Run dtests (j8_upgradetests_without_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_upgradetests_without_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_upgradetests_without_vnodes_logs - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6830,95 +7027,6 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_jvm_upgrade_dtests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Repeatedly run new or modifed JUnit tests - no_output_timeout: 15m - command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s///\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" - - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utests/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utests/logs - destination: logs - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -10213,109 +10321,6 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_dtest_jars_build: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Build Cassandra DTest jars - command: | - export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - mkdir ~/dtest_jars - git remote add apache https://github.com/apache/cassandra.git - for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 cassandra-4.1 trunk; do - # check out the correct cassandra version: - git remote set-branches --add apache '$branch' - git fetch --depth 1 apache $branch - git checkout $branch - git clean -fd - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - cp build/dtest*.jar ~/dtest_jars - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - done - # and build the dtest-jar for the branch under test - ${ANT_HOME}/bin/ant realclean - git checkout origin/$CIRCLE_BRANCH - git clean -fd - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - cp build/dtest*.jar ~/dtest_jars - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - ls -l ~/dtest_jars - no_output_timeout: 15m - - persist_to_workspace: - root: /home/cassandra - paths: - - dtest_jars - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 java8_separate_tests: @@ -10483,18 +10488,6 @@ workflows: requires: - start_j11_utests_system_keyspace_directory - j8_build - - start_j8_dtest_jars_build: - type: approval - - j8_dtest_jars_build: - requires: - - j8_build - - start_j8_dtest_jars_build - - start_jvm_upgrade_dtests: - type: approval - - j8_jvm_upgrade_dtests: - requires: - - start_jvm_upgrade_dtests - - j8_dtest_jars_build - start_j8_dtests: type: approval - j8_dtests: @@ -10555,12 +10548,6 @@ workflows: requires: - start_j11_dtests_large_vnode - j8_build - - start_upgrade_dtests: - type: approval - - j8_upgrade_dtests: - requires: - - start_upgrade_dtests - - j8_build - start_j8_cqlsh_tests: type: approval - j8_cqlsh_dtests_py3: @@ -10756,15 +10743,6 @@ workflows: requires: - start_utests_system_keyspace_directory - j8_build - - start_jvm_upgrade_dtests: - type: approval - - j8_dtest_jars_build: - requires: - - j8_build - - start_jvm_upgrade_dtests - - j8_jvm_upgrade_dtests: - requires: - - j8_dtest_jars_build - j8_dtests: requires: - j8_build @@ -10809,12 +10787,6 @@ workflows: requires: - start_j11_dtests_large - j8_build - - start_upgrade_dtests: - type: approval - - j8_upgrade_dtests: - requires: - - j8_build - - start_upgrade_dtests - j8_cqlsh_dtests_py3: requires: - j8_build @@ -11038,6 +11010,24 @@ workflows: requires: - start_j11_utests_system_keyspace_directory - j11_build + - start_j11_dtest_jars_build: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_j11_dtest_jars_build + - start_jvm_upgrade_dtests: + type: approval + - j11_jvm_upgrade_dtests: + requires: + - start_jvm_upgrade_dtests + - j11_dtest_jars_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - start_j11_upgrade_dtests + - j11_build java11_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -11162,3 +11152,18 @@ workflows: requires: - start_utests_system_keyspace_directory - j11_build + - start_jvm_upgrade_dtests: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_jvm_upgrade_dtests + - j11_jvm_upgrade_dtests: + requires: + - j11_dtest_jars_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - j11_build + - start_j11_upgrade_dtests diff --git a/.circleci/config.yml.FREE b/.circleci/config.yml.FREE index 7ce806829c..df8682ec52 100644 --- a/.circleci/config.yml.FREE +++ b/.circleci/config.yml.FREE @@ -18,35 +18,16 @@ version: 2 jobs: - j8_jvm_upgrade_dtests: + j11_jvm_upgrade_dtests_repeat: docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + - image: apache/cassandra-testing-ubuntu2004-java11:latest resource_class: medium working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 1 + parallelism: 4 steps: - attach_workspace: at: /home/cassandra - - run: - name: Determine distributed Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - run: name: Log Environment Information command: | @@ -70,28 +51,19 @@ jobs: which java java -version - run: - name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + name: Repeatedly run new or modifed JUnit tests no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s///\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" - store_test_results: - path: /tmp/cassandra/build/test/output/ + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/cassandra/build/test/output + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output destination: junitxml - store_artifacts: - path: /tmp/cassandra/build/test/logs + path: /tmp/results/repeated_utests/logs destination: logs environment: - ANT_HOME: /usr/share/ant @@ -133,8 +105,9 @@ jobs: - REPEATED_ANT_TEST_METHODS: null - REPEATED_ANT_TEST_VNODES: false - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_fqltool_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2668,6 +2641,139 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Run repeated Python DTests + no_output_timeout: 15m + command: | + if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if true; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_cqlsh_dtests_py3_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4000,6 +4106,110 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_dtest_jars_build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + mkdir ~/dtest_jars + git remote add apache https://github.com/apache/cassandra.git + for branch in cassandra-4.0 cassandra-4.1 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + git clean -fd + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + ${ANT_HOME}/bin/ant realclean + git checkout origin/$CIRCLE_BRANCH + git clean -fd + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_long: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4359,6 +4569,124 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_jvm_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_stress: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4539,138 +4867,6 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgrade_dtests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Run repeated Python DTests - no_output_timeout: 15m - command: | - if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") - - stop_on_failure_arg="" - if ${REPEATED_TESTS_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if false; then - vnodes_args="--use-vnodes --num-tokens=16" - fi - - upgrade_arg="" - if true; then - upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_utests_cdc_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5951,6 +6147,91 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_upgradetests_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j11_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j11_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_upgradetests_without_vnodes > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_upgradetests_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_upgradetests_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_upgradetests_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6639,90 +6920,6 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_upgrade_dtests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j8_upgradetests_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n" - - run: - name: Run dtests (j8_upgradetests_without_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_upgradetests_without_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_upgradetests_without_vnodes_logs - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6830,95 +7027,6 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_jvm_upgrade_dtests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Repeatedly run new or modifed JUnit tests - no_output_timeout: 15m - command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s///\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" - - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utests/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utests/logs - destination: logs - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -10213,109 +10321,6 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_dtest_jars_build: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Build Cassandra DTest jars - command: | - export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - mkdir ~/dtest_jars - git remote add apache https://github.com/apache/cassandra.git - for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 cassandra-4.1 trunk; do - # check out the correct cassandra version: - git remote set-branches --add apache '$branch' - git fetch --depth 1 apache $branch - git checkout $branch - git clean -fd - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - cp build/dtest*.jar ~/dtest_jars - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - done - # and build the dtest-jar for the branch under test - ${ANT_HOME}/bin/ant realclean - git checkout origin/$CIRCLE_BRANCH - git clean -fd - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - cp build/dtest*.jar ~/dtest_jars - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - ls -l ~/dtest_jars - no_output_timeout: 15m - - persist_to_workspace: - root: /home/cassandra - paths: - - dtest_jars - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 java8_separate_tests: @@ -10483,18 +10488,6 @@ workflows: requires: - start_j11_utests_system_keyspace_directory - j8_build - - start_j8_dtest_jars_build: - type: approval - - j8_dtest_jars_build: - requires: - - j8_build - - start_j8_dtest_jars_build - - start_jvm_upgrade_dtests: - type: approval - - j8_jvm_upgrade_dtests: - requires: - - start_jvm_upgrade_dtests - - j8_dtest_jars_build - start_j8_dtests: type: approval - j8_dtests: @@ -10555,12 +10548,6 @@ workflows: requires: - start_j11_dtests_large_vnode - j8_build - - start_upgrade_dtests: - type: approval - - j8_upgrade_dtests: - requires: - - start_upgrade_dtests - - j8_build - start_j8_cqlsh_tests: type: approval - j8_cqlsh_dtests_py3: @@ -10756,15 +10743,6 @@ workflows: requires: - start_utests_system_keyspace_directory - j8_build - - start_jvm_upgrade_dtests: - type: approval - - j8_dtest_jars_build: - requires: - - j8_build - - start_jvm_upgrade_dtests - - j8_jvm_upgrade_dtests: - requires: - - j8_dtest_jars_build - j8_dtests: requires: - j8_build @@ -10809,12 +10787,6 @@ workflows: requires: - start_j11_dtests_large - j8_build - - start_upgrade_dtests: - type: approval - - j8_upgrade_dtests: - requires: - - j8_build - - start_upgrade_dtests - j8_cqlsh_dtests_py3: requires: - j8_build @@ -11038,6 +11010,24 @@ workflows: requires: - start_j11_utests_system_keyspace_directory - j11_build + - start_j11_dtest_jars_build: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_j11_dtest_jars_build + - start_jvm_upgrade_dtests: + type: approval + - j11_jvm_upgrade_dtests: + requires: + - start_jvm_upgrade_dtests + - j11_dtest_jars_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - start_j11_upgrade_dtests + - j11_build java11_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -11162,3 +11152,18 @@ workflows: requires: - start_utests_system_keyspace_directory - j11_build + - start_jvm_upgrade_dtests: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_jvm_upgrade_dtests + - j11_jvm_upgrade_dtests: + requires: + - j11_dtest_jars_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - j11_build + - start_j11_upgrade_dtests diff --git a/.circleci/config.yml.PAID b/.circleci/config.yml.PAID index 0aa2cfde4f..f100c2274e 100644 --- a/.circleci/config.yml.PAID +++ b/.circleci/config.yml.PAID @@ -18,35 +18,16 @@ version: 2 jobs: - j8_jvm_upgrade_dtests: + j11_jvm_upgrade_dtests_repeat: docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: large working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 4 + parallelism: 25 steps: - attach_workspace: at: /home/cassandra - - run: - name: Determine distributed Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - run: name: Log Environment Information command: | @@ -70,28 +51,19 @@ jobs: which java java -version - run: - name: Run Unit Tests (testclasslist) - command: | - set -x - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - if [ -d ~/dtest_jars ]; then - cp ~/dtest_jars/dtest* /tmp/cassandra/build/ - fi - test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) - if [ -z "$test_timeout" ]; then - test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') - fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + name: Repeatedly run new or modifed JUnit tests no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s///\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" - store_test_results: - path: /tmp/cassandra/build/test/output/ + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/cassandra/build/test/output + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output destination: junitxml - store_artifacts: - path: /tmp/cassandra/build/test/logs + path: /tmp/results/repeated_utests/logs destination: logs environment: - ANT_HOME: /usr/share/ant @@ -133,8 +105,9 @@ jobs: - REPEATED_ANT_TEST_METHODS: null - REPEATED_ANT_TEST_VNODES: false - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_fqltool_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2668,6 +2641,139 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Run repeated Python DTests + no_output_timeout: 15m + command: | + if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if true; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_cqlsh_dtests_py3_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4000,6 +4106,110 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_dtest_jars_build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + mkdir ~/dtest_jars + git remote add apache https://github.com/apache/cassandra.git + for branch in cassandra-4.0 cassandra-4.1 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + git clean -fd + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + ${ANT_HOME}/bin/ant realclean + git checkout origin/$CIRCLE_BRANCH + git clean -fd + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_long: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4359,6 +4569,124 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_jvm_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_stress: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4539,138 +4867,6 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_upgrade_dtests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 25 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Run repeated Python DTests - no_output_timeout: 15m - command: | - if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then - echo "Repeated dtest name hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then - echo "Repeated dtest count hasn't been defined, exiting without running any test" - elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then - echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" - else - - # Calculate the number of test iterations to be run by the current parallel runner. - # Since we are running the same test multiple times there is no need to use `circleci tests split`. - count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - if (($count <= 0)); then - echo "No tests to run in this runner" - else - echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - - java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - - tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") - - stop_on_failure_arg="" - if ${REPEATED_TESTS_STOP_ON_FAILURE}; then - stop_on_failure_arg="-x" - fi - - vnodes_args="" - if false; then - vnodes_args="--use-vnodes --num-tokens=16" - fi - - upgrade_arg="" - if true; then - upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" - fi - - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt - fi - fi - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_logs - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_utests_cdc_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5951,6 +6147,91 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_upgradetests_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j11_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j11_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_upgradetests_without_vnodes > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_upgradetests_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_upgradetests_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_upgradetests_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6639,90 +6920,6 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_upgrade_dtests: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: large - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j8_upgradetests_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j8_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n" - - run: - name: Run dtests (j8_upgradetests_without_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_upgradetests_without_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_upgradetests_without_vnodes_logs - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_dtests_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6830,95 +7027,6 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_jvm_upgrade_dtests_repeat: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: large - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 25 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Repeatedly run new or modifed JUnit tests - no_output_timeout: 15m - command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s///\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" - - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout - - store_artifacts: - path: /tmp/results/repeated_utests/output - destination: junitxml - - store_artifacts: - path: /tmp/results/repeated_utests/logs - destination: logs - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_cqlsh_dtests_py38_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -10213,109 +10321,6 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true - j8_dtest_jars_build: - docker: - - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Build Cassandra DTest jars - command: | - export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - mkdir ~/dtest_jars - git remote add apache https://github.com/apache/cassandra.git - for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 cassandra-4.1 trunk; do - # check out the correct cassandra version: - git remote set-branches --add apache '$branch' - git fetch --depth 1 apache $branch - git checkout $branch - git clean -fd - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - cp build/dtest*.jar ~/dtest_jars - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - done - # and build the dtest-jar for the branch under test - ${ANT_HOME}/bin/ant realclean - git checkout origin/$CIRCLE_BRANCH - git clean -fd - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - cp build/dtest*.jar ~/dtest_jars - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi - ls -l ~/dtest_jars - no_output_timeout: 15m - - persist_to_workspace: - root: /home/cassandra - paths: - - dtest_jars - environment: - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: https://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: trunk - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - REPEATED_TESTS_STOP_ON_FAILURE: false - - REPEATED_UTESTS: null - - REPEATED_UTESTS_COUNT: 500 - - REPEATED_UTESTS_FQLTOOL: null - - REPEATED_UTESTS_FQLTOOL_COUNT: 500 - - REPEATED_UTESTS_LONG: null - - REPEATED_UTESTS_LONG_COUNT: 100 - - REPEATED_UTESTS_STRESS: null - - REPEATED_UTESTS_STRESS_COUNT: 500 - - REPEATED_SIMULATOR_DTESTS: null - - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - - REPEATED_JVM_DTESTS: null - - REPEATED_JVM_DTESTS_COUNT: 500 - - REPEATED_JVM_UPGRADE_DTESTS: null - - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - - REPEATED_DTESTS: null - - REPEATED_DTESTS_COUNT: 500 - - REPEATED_LARGE_DTESTS: null - - REPEATED_LARGE_DTESTS_COUNT: 100 - - REPEATED_UPGRADE_DTESTS: null - - REPEATED_UPGRADE_DTESTS_COUNT: 25 - - REPEATED_ANT_TEST_TARGET: testsome - - REPEATED_ANT_TEST_CLASS: null - - REPEATED_ANT_TEST_METHODS: null - - REPEATED_ANT_TEST_VNODES: false - - REPEATED_ANT_TEST_COUNT: 500 - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 java8_separate_tests: @@ -10483,18 +10488,6 @@ workflows: requires: - start_j11_utests_system_keyspace_directory - j8_build - - start_j8_dtest_jars_build: - type: approval - - j8_dtest_jars_build: - requires: - - j8_build - - start_j8_dtest_jars_build - - start_jvm_upgrade_dtests: - type: approval - - j8_jvm_upgrade_dtests: - requires: - - start_jvm_upgrade_dtests - - j8_dtest_jars_build - start_j8_dtests: type: approval - j8_dtests: @@ -10555,12 +10548,6 @@ workflows: requires: - start_j11_dtests_large_vnode - j8_build - - start_upgrade_dtests: - type: approval - - j8_upgrade_dtests: - requires: - - start_upgrade_dtests - - j8_build - start_j8_cqlsh_tests: type: approval - j8_cqlsh_dtests_py3: @@ -10756,15 +10743,6 @@ workflows: requires: - start_utests_system_keyspace_directory - j8_build - - start_jvm_upgrade_dtests: - type: approval - - j8_dtest_jars_build: - requires: - - j8_build - - start_jvm_upgrade_dtests - - j8_jvm_upgrade_dtests: - requires: - - j8_dtest_jars_build - j8_dtests: requires: - j8_build @@ -10809,12 +10787,6 @@ workflows: requires: - start_j11_dtests_large - j8_build - - start_upgrade_dtests: - type: approval - - j8_upgrade_dtests: - requires: - - j8_build - - start_upgrade_dtests - j8_cqlsh_dtests_py3: requires: - j8_build @@ -11038,6 +11010,24 @@ workflows: requires: - start_j11_utests_system_keyspace_directory - j11_build + - start_j11_dtest_jars_build: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_j11_dtest_jars_build + - start_jvm_upgrade_dtests: + type: approval + - j11_jvm_upgrade_dtests: + requires: + - start_jvm_upgrade_dtests + - j11_dtest_jars_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - start_j11_upgrade_dtests + - j11_build java11_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -11162,3 +11152,18 @@ workflows: requires: - start_utests_system_keyspace_directory - j11_build + - start_jvm_upgrade_dtests: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_jvm_upgrade_dtests + - j11_jvm_upgrade_dtests: + requires: + - j11_dtest_jars_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - j11_build + - start_j11_upgrade_dtests diff --git a/.circleci/config_11_and_17.yml b/.circleci/config_11_and_17.yml index cb507ef725..77866629cc 100644 --- a/.circleci/config_11_and_17.yml +++ b/.circleci/config_11_and_17.yml @@ -18,6 +18,96 @@ version: 2 jobs: + j11_jvm_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s///\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -109,10 +199,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -168,10 +262,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -272,10 +370,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -357,10 +459,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -442,10 +548,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -453,6 +563,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlshlib_cython_tests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -501,10 +612,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -512,6 +627,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py3_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -605,10 +721,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -718,10 +838,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -822,10 +946,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -972,10 +1100,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1100,10 +1232,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1228,10 +1364,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1239,6 +1379,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_system_keyspace_directory: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1341,10 +1482,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1352,6 +1497,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py3: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -1445,10 +1591,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1530,10 +1680,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1541,6 +1695,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_unit_tests: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -1643,10 +1798,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1728,10 +1887,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1832,10 +1995,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1843,6 +2010,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_dtests_large_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -1960,10 +2128,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2110,10 +2282,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2223,10 +2399,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2234,6 +2414,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py311: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2327,10 +2508,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2431,10 +2616,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2442,6 +2631,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_stress_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2516,10 +2706,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2601,10 +2795,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2612,6 +2810,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_oa: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2714,10 +2913,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2905,10 +3108,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2916,6 +3123,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -2985,10 +3193,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2996,6 +3208,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3113,10 +3326,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3124,6 +3341,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_trie: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -3226,10 +3444,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3330,10 +3552,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3341,6 +3567,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3410,10 +3637,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3421,6 +3652,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3495,10 +3727,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3506,6 +3742,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_oa: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3608,10 +3845,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3619,6 +3860,140 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Run repeated Python DTests + no_output_timeout: 15m + command: | + if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if true; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_jvm_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -3721,10 +4096,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3912,10 +4291,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4016,10 +4399,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4027,6 +4414,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_build: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4118,10 +4506,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4231,10 +4623,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4242,6 +4638,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_cdc_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4316,10 +4713,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4383,10 +4784,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4394,6 +4799,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_offheap_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4511,10 +4917,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4522,6 +4932,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_jvm_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4624,10 +5035,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4635,6 +5050,111 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_dtest_jars_build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + mkdir ~/dtest_jars + git remote add apache https://github.com/apache/cassandra.git + for branch in cassandra-4.0 cassandra-4.1 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + git clean -fd + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + ${ANT_HOME}/bin/ant realclean + git checkout origin/$CIRCLE_BRANCH + git clean -fd + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_long: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4691,10 +5211,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4702,6 +5226,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_unit_tests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4776,10 +5301,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4885,10 +5414,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4896,6 +5429,125 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_jvm_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_stress: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4952,10 +5604,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4963,6 +5619,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_cdc_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5037,10 +5694,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5048,6 +5709,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5139,10 +5801,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5267,10 +5933,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5278,6 +5948,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_compression: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5380,10 +6051,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5391,6 +6066,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_trie_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5465,10 +6141,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5569,10 +6249,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5580,6 +6264,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlsh_dtests_py311_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5673,10 +6358,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5684,6 +6373,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_trie: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5786,10 +6476,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5797,6 +6491,92 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_upgradetests_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j11_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j11_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_upgradetests_without_vnodes > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_upgradetests_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_upgradetests_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_upgradetests_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5914,10 +6694,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5925,6 +6709,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5999,10 +6784,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6103,10 +6892,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6114,6 +6907,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_fqltool: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6170,10 +6964,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6250,10 +7048,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6330,10 +7132,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6341,6 +7147,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlsh_dtests_py38_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6434,10 +7241,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6445,6 +7256,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_trie_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6519,10 +7331,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6530,6 +7346,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_simulator_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6604,10 +7421,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6615,6 +7436,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_jvm_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6689,10 +7511,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6700,6 +7526,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6793,10 +7620,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6851,10 +7682,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6955,10 +7790,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6966,6 +7805,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_system_keyspace_directory: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -7068,10 +7908,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7172,10 +8016,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7276,10 +8124,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7389,10 +8241,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7400,6 +8256,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_stress: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -7456,10 +8313,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7541,10 +8402,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7552,6 +8417,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_build: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7643,10 +8509,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7654,6 +8524,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_oa_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7728,10 +8599,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7739,6 +8614,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlshlib_tests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7786,10 +8662,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7797,6 +8677,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7866,10 +8747,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7877,6 +8762,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_jvm_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -7979,10 +8865,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8083,10 +8973,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8196,10 +9090,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8281,10 +9179,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8361,10 +9263,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8446,10 +9352,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8457,6 +9367,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_long: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -8513,10 +9424,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8598,10 +9513,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8678,10 +9597,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8689,6 +9612,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_jvm_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -8763,10 +9687,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8865,10 +9793,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8993,10 +9925,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -9078,10 +10014,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -9163,10 +10103,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -9174,6 +10118,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true workflows: version: 2 java11_separate_tests: @@ -9341,6 +10286,18 @@ workflows: requires: - start_j17_utests_system_keyspace_directory - j11_build + - start_j11_dtest_jars_build: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_j11_dtest_jars_build + - start_jvm_upgrade_dtests: + type: approval + - j11_jvm_upgrade_dtests: + requires: + - start_jvm_upgrade_dtests + - j11_dtest_jars_build - start_j11_dtests: type: approval - j11_dtests: @@ -9481,6 +10438,12 @@ workflows: requires: - start_j17_cqlsh_tests_offheap - j11_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - start_j11_upgrade_dtests + - j11_build java11_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9596,6 +10559,15 @@ workflows: requires: - start_utests_system_keyspace_directory - j11_build + - start_jvm_upgrade_dtests: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_jvm_upgrade_dtests + - j11_jvm_upgrade_dtests: + requires: + - j11_dtest_jars_build - j11_dtests: requires: - j11_build @@ -9704,6 +10676,12 @@ workflows: requires: - start_j17_cqlsh-dtests-offheap - j11_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - j11_build + - start_j11_upgrade_dtests java17_separate_tests: jobs: - start_j17_build: diff --git a/.circleci/config_11_and_17.yml.FREE b/.circleci/config_11_and_17.yml.FREE index cb507ef725..77866629cc 100644 --- a/.circleci/config_11_and_17.yml.FREE +++ b/.circleci/config_11_and_17.yml.FREE @@ -18,6 +18,96 @@ version: 2 jobs: + j11_jvm_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s///\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -109,10 +199,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -168,10 +262,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -272,10 +370,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -357,10 +459,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -442,10 +548,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -453,6 +563,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlshlib_cython_tests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -501,10 +612,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -512,6 +627,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py3_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -605,10 +721,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -718,10 +838,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -822,10 +946,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -972,10 +1100,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1100,10 +1232,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1228,10 +1364,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1239,6 +1379,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_system_keyspace_directory: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1341,10 +1482,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1352,6 +1497,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py3: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -1445,10 +1591,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1530,10 +1680,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1541,6 +1695,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_unit_tests: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -1643,10 +1798,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1728,10 +1887,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1832,10 +1995,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1843,6 +2010,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_dtests_large_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -1960,10 +2128,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2110,10 +2282,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2223,10 +2399,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2234,6 +2414,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py311: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2327,10 +2508,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2431,10 +2616,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2442,6 +2631,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_stress_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2516,10 +2706,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2601,10 +2795,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2612,6 +2810,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_oa: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2714,10 +2913,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2905,10 +3108,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2916,6 +3123,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -2985,10 +3193,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2996,6 +3208,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3113,10 +3326,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3124,6 +3341,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_trie: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -3226,10 +3444,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3330,10 +3552,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3341,6 +3567,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3410,10 +3637,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3421,6 +3652,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3495,10 +3727,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3506,6 +3742,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_oa: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3608,10 +3845,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3619,6 +3860,140 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Run repeated Python DTests + no_output_timeout: 15m + command: | + if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if true; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_jvm_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -3721,10 +4096,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3912,10 +4291,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4016,10 +4399,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4027,6 +4414,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_build: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4118,10 +4506,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4231,10 +4623,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4242,6 +4638,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_cdc_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4316,10 +4713,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4383,10 +4784,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4394,6 +4799,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_offheap_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4511,10 +4917,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4522,6 +4932,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_jvm_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4624,10 +5035,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4635,6 +5050,111 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_dtest_jars_build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + mkdir ~/dtest_jars + git remote add apache https://github.com/apache/cassandra.git + for branch in cassandra-4.0 cassandra-4.1 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + git clean -fd + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + ${ANT_HOME}/bin/ant realclean + git checkout origin/$CIRCLE_BRANCH + git clean -fd + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_long: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4691,10 +5211,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4702,6 +5226,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_unit_tests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4776,10 +5301,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4885,10 +5414,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4896,6 +5429,125 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_jvm_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_stress: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4952,10 +5604,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4963,6 +5619,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_cdc_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5037,10 +5694,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5048,6 +5709,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5139,10 +5801,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5267,10 +5933,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5278,6 +5948,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_compression: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5380,10 +6051,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5391,6 +6066,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_trie_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5465,10 +6141,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5569,10 +6249,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5580,6 +6264,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlsh_dtests_py311_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5673,10 +6358,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5684,6 +6373,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_trie: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5786,10 +6476,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5797,6 +6491,92 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_upgradetests_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j11_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j11_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_upgradetests_without_vnodes > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_upgradetests_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_upgradetests_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_upgradetests_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5914,10 +6694,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5925,6 +6709,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5999,10 +6784,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6103,10 +6892,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6114,6 +6907,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_fqltool: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6170,10 +6964,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6250,10 +7048,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6330,10 +7132,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6341,6 +7147,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlsh_dtests_py38_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6434,10 +7241,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6445,6 +7256,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_trie_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6519,10 +7331,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6530,6 +7346,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_simulator_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6604,10 +7421,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6615,6 +7436,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_jvm_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6689,10 +7511,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6700,6 +7526,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6793,10 +7620,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6851,10 +7682,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6955,10 +7790,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6966,6 +7805,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_system_keyspace_directory: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -7068,10 +7908,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7172,10 +8016,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7276,10 +8124,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7389,10 +8241,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7400,6 +8256,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_stress: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -7456,10 +8313,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7541,10 +8402,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7552,6 +8417,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_build: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7643,10 +8509,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7654,6 +8524,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_oa_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7728,10 +8599,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7739,6 +8614,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlshlib_tests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7786,10 +8662,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7797,6 +8677,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7866,10 +8747,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7877,6 +8762,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_jvm_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -7979,10 +8865,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8083,10 +8973,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8196,10 +9090,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8281,10 +9179,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8361,10 +9263,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8446,10 +9352,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8457,6 +9367,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_long: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -8513,10 +9424,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8598,10 +9513,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8678,10 +9597,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8689,6 +9612,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_jvm_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -8763,10 +9687,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8865,10 +9793,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8993,10 +9925,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -9078,10 +10014,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -9163,10 +10103,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -9174,6 +10118,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true workflows: version: 2 java11_separate_tests: @@ -9341,6 +10286,18 @@ workflows: requires: - start_j17_utests_system_keyspace_directory - j11_build + - start_j11_dtest_jars_build: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_j11_dtest_jars_build + - start_jvm_upgrade_dtests: + type: approval + - j11_jvm_upgrade_dtests: + requires: + - start_jvm_upgrade_dtests + - j11_dtest_jars_build - start_j11_dtests: type: approval - j11_dtests: @@ -9481,6 +10438,12 @@ workflows: requires: - start_j17_cqlsh_tests_offheap - j11_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - start_j11_upgrade_dtests + - j11_build java11_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9596,6 +10559,15 @@ workflows: requires: - start_utests_system_keyspace_directory - j11_build + - start_jvm_upgrade_dtests: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_jvm_upgrade_dtests + - j11_jvm_upgrade_dtests: + requires: + - j11_dtest_jars_build - j11_dtests: requires: - j11_build @@ -9704,6 +10676,12 @@ workflows: requires: - start_j17_cqlsh-dtests-offheap - j11_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - j11_build + - start_j11_upgrade_dtests java17_separate_tests: jobs: - start_j17_build: diff --git a/.circleci/config_11_and_17.yml.PAID b/.circleci/config_11_and_17.yml.PAID index 8b9cbec2c1..d67a3d08d3 100644 --- a/.circleci/config_11_and_17.yml.PAID +++ b/.circleci/config_11_and_17.yml.PAID @@ -18,6 +18,96 @@ version: 2 jobs: + j11_jvm_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically detected tests together, removing duplicates\ntests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e \"s///\" | sed -e \"s/ //\" | tr \",\" \"\\n\" | tr \" \" \"\\n\" | sort -n | uniq -u)\necho \"Tests to be repeated: ${tests}\"\n\n# Prepare the JVM dtests vnodes argument, which is optional.\nvnodes=false\nvnodes_args=\"\"\nif [ \"$vnodes\" = true ] ; then\n vnodes_args=\"-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'\"\nfi\n\n# Prepare the testtag for the target, used by the test macro in build.xml to group the output files\ntarget=test-jvm-dtest-some\ntesttag=\"\"\nif [[ $target == \"test-cdc\" ]]; then\n testtag=\"cdc\"\nelif [[ $target == \"test-compression\" ]]; then\n testtag=\"compression\"\nelif [[ $target == \"test-system-keyspace-directory\" ]]; then\n testtag=\"system_keyspace_directory\"\nelif [[ $target == \"test-trie\" ]]; then\n testtag=\"trie\"\nelif [[ $target == \"test-oa\" ]]; then\n testtag=\"oa\"\nfi\n\n# Run each test class as many times as requested.\nexit_code=\"$?\"\nfor test in $tests; do\n\n # Split class and method names from the test name\n if [[ $test =~ \"#\" ]]; then\n class=${test%\"#\"*}\n method=${test#*\"#\"}\n else\n class=$test\n method=\"\"\n fi\n\n # Prepare the -Dtest.name argument.\n # It can be the fully qualified class name or the short class name, depending on the target.\n if [[ $target == \"test\" || \\\n $target == \"test-cdc\" || \\\n $target == \"test-compression\" || \\\n $target == \"test-trie\" || \\\n $target == \"test-oa\" || \\\n $target == \"test-system-keyspace-directory\" || \\\n $target == \"fqltool-test\" || \\\n $target == \"long-test\" || \\\n $target == \"stress-test\" || \\\n $target == \"test-simulator-dtest\" ]]; then\n name_arg=\"-Dtest.name=${class##*.}\"\n else\n name_arg=\"-Dtest.name=$class\"\n fi\n\n # Prepare the -Dtest.methods argument, which is optional\n if [[ $method == \"\" ]]; then\n methods_arg=\"\"\n else\n methods_arg=\"-Dtest.methods=$method\"\n fi\n\n for i in $(seq -w 1 $count); do\n echo \"Running test $test, iteration $i of $count\"\n\n # run the test\n status=\"passes\"\n if !( set -o pipefail && \\\n ant test-jvm-dtest-some $name_arg $methods_arg $vnodes_args -Dno-build-test=true | \\\n tee stdout.txt \\\n ); then\n status=\"fails\"\n exit_code=1\n fi\n\n # move the stdout output file\n dest=/tmp/results/repeated_utests/stdout/${status}/${i}\n mkdir -p $dest\n mv stdout.txt $dest/${test}.txt\n\n # move the XML output files\n source=build/test/output/${testtag}\n dest=/tmp/results/repeated_utests/output/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n\n # move the log files\n source=build/test/logs/${testtag}\n dest=/tmp/results/repeated_utests/logs/${status}/${i}\n mkdir -p $dest\n if [[ -d $source && -n \"$(ls $source)\" ]]; then\n mv $source/* $dest/\n fi\n \n # maybe stop iterations on test failure\n if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then\n break\n fi\n done\ndone\n(exit ${exit_code})\n" + - store_test_results: + path: /tmp/results/repeated_utests/output + - store_artifacts: + path: /tmp/results/repeated_utests/stdout + destination: stdout + - store_artifacts: + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -109,10 +199,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -168,10 +262,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -272,10 +370,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -357,10 +459,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -442,10 +548,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -453,6 +563,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlshlib_cython_tests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -501,10 +612,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -512,6 +627,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py3_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -605,10 +721,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -718,10 +838,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -822,10 +946,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -972,10 +1100,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1100,10 +1232,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1228,10 +1364,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1239,6 +1379,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_system_keyspace_directory: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -1341,10 +1482,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1352,6 +1497,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py3: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -1445,10 +1591,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1530,10 +1680,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1541,6 +1695,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_unit_tests: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -1643,10 +1798,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1728,10 +1887,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1832,10 +1995,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -1843,6 +2010,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_dtests_large_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -1960,10 +2128,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2110,10 +2282,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2223,10 +2399,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2234,6 +2414,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py311: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2327,10 +2508,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2431,10 +2616,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2442,6 +2631,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_stress_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2516,10 +2706,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2601,10 +2795,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2612,6 +2810,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_oa: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -2714,10 +2913,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2905,10 +3108,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2916,6 +3123,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -2985,10 +3193,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -2996,6 +3208,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_vnode_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3113,10 +3326,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3124,6 +3341,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_trie: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -3226,10 +3444,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3330,10 +3552,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3341,6 +3567,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3410,10 +3637,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3421,6 +3652,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_system_keyspace_directory_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3495,10 +3727,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3506,6 +3742,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_oa: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -3608,10 +3845,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3619,6 +3860,140 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Run repeated Python DTests + no_output_timeout: 15m + command: | + if [ "${REPEATED_UPGRADE_DTESTS}" == "" ]; then + echo "Repeated dtest name hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "" ]; then + echo "Repeated dtest count hasn't been defined, exiting without running any test" + elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then + echo "Repeated dtest count is lesser or equals than zero, exiting without running any test" + else + + # Calculate the number of test iterations to be run by the current parallel runner. + # Since we are running the same test multiple times there is no need to use `circleci tests split`. + count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL)) + if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then + count=$((count+1)) + fi + + if (($count <= 0)); then + echo "No tests to run in this runner" + else + echo "Running ${REPEATED_UPGRADE_DTESTS} $count times" + + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + + tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g") + + stop_on_failure_arg="" + if ${REPEATED_TESTS_STOP_ON_FAILURE}; then + stop_on_failure_arg="-x" + fi + + vnodes_args="" + if false; then + vnodes_args="--use-vnodes --num-tokens=16" + fi + + upgrade_arg="" + if true; then + upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all" + fi + + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt + fi + fi + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_jvm_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -3721,10 +4096,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -3912,10 +4291,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4016,10 +4399,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4027,6 +4414,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_build: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4118,10 +4506,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4231,10 +4623,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4242,6 +4638,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_cdc_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4316,10 +4713,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4383,10 +4784,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4394,6 +4799,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_offheap_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4511,10 +4917,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4522,6 +4932,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_jvm_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4624,10 +5035,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4635,6 +5050,111 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_dtest_jars_build: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + mkdir ~/dtest_jars + git remote add apache https://github.com/apache/cassandra.git + for branch in cassandra-4.0 cassandra-4.1 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + git clean -fd + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + ${ANT_HOME}/bin/ant realclean + git checkout origin/$CIRCLE_BRANCH + git clean -fd + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant realclean; ${ANT_HOME}/bin/ant jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + cp build/dtest*.jar ~/dtest_jars + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_long: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4691,10 +5211,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4702,6 +5226,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_unit_tests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -4776,10 +5301,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4885,10 +5414,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4896,6 +5429,125 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_jvm_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') + fi + ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_stress: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -4952,10 +5604,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -4963,6 +5619,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_cdc_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5037,10 +5694,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5048,6 +5709,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5139,10 +5801,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5267,10 +5933,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5278,6 +5948,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_compression: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5380,10 +6051,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5391,6 +6066,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_trie_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5465,10 +6141,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5569,10 +6249,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5580,6 +6264,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlsh_dtests_py311_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5673,10 +6358,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5684,6 +6373,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_trie: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5786,10 +6476,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5797,6 +6491,92 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_upgrade_dtests: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: large + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env3.6/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_upgradetests_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw /tmp/all_dtest_tests_j11_upgradetests_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_upgradetests_without_vnodes_raw > /tmp/all_dtest_tests_j11_upgradetests_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_upgradetests_without_vnodes > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_upgradetests_without_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_upgradetests_without_vnodes_final.txt`\nif [ ! -z \"$SPLIT_TESTS\" ]; then\n set -o pipefail && cd ~/cassandra-dtest && pytest --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --log-level=\"DEBUG\" --junit-xml=/tmp/results/dtests/pytest_result_j11_upgradetests_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\nelse\n echo \"Tune your parallelism, there are more containers than test classes. Nothing to do in this container\"\n (exit 1)\nfi\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_upgradetests_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_upgradetests_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_LARGE_DTESTS: null + - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests_large_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -5914,10 +6694,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -5925,6 +6709,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_compression_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -5999,10 +6784,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6103,10 +6892,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6114,6 +6907,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_fqltool: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6170,10 +6964,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6250,10 +7048,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6330,10 +7132,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6341,6 +7147,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlsh_dtests_py38_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6434,10 +7241,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6445,6 +7256,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_trie_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6519,10 +7331,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6530,6 +7346,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_simulator_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6604,10 +7421,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6615,6 +7436,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_jvm_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -6689,10 +7511,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6700,6 +7526,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_cqlsh_dtests_py38_offheap: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -6793,10 +7620,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6851,10 +7682,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6955,10 +7790,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -6966,6 +7805,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_system_keyspace_directory: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -7068,10 +7908,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7172,10 +8016,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7276,10 +8124,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7389,10 +8241,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7400,6 +8256,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_stress: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -7456,10 +8313,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7541,10 +8402,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7552,6 +8417,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_build: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7643,10 +8509,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7654,6 +8524,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_utests_oa_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7728,10 +8599,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7739,6 +8614,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_cqlshlib_tests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7786,10 +8662,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7797,6 +8677,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests: docker: - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest @@ -7866,10 +8747,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -7877,6 +8762,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_jvm_dtests_vnode: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -7979,10 +8865,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8083,10 +8973,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8196,10 +9090,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8281,10 +9179,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8361,10 +9263,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8446,10 +9352,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8457,6 +9367,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_utests_long: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -8513,10 +9424,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8598,10 +9513,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8678,10 +9597,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8689,6 +9612,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j17_jvm_dtests_repeat: docker: - image: apache/cassandra-testing-ubuntu2004-java11:latest @@ -8763,10 +9687,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8865,10 +9793,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -8993,10 +9925,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -9078,10 +10014,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -9163,10 +10103,14 @@ jobs: - REPEATED_SIMULATOR_DTESTS_COUNT: 500 - REPEATED_JVM_DTESTS: null - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 - REPEATED_DTESTS: null - REPEATED_DTESTS_COUNT: 500 - REPEATED_LARGE_DTESTS: null - REPEATED_LARGE_DTESTS_COUNT: 100 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 - REPEATED_ANT_TEST_TARGET: testsome - REPEATED_ANT_TEST_CLASS: null - REPEATED_ANT_TEST_METHODS: null @@ -9174,6 +10118,7 @@ jobs: - REPEATED_ANT_TEST_COUNT: 500 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true workflows: version: 2 java11_separate_tests: @@ -9341,6 +10286,18 @@ workflows: requires: - start_j17_utests_system_keyspace_directory - j11_build + - start_j11_dtest_jars_build: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_j11_dtest_jars_build + - start_jvm_upgrade_dtests: + type: approval + - j11_jvm_upgrade_dtests: + requires: + - start_jvm_upgrade_dtests + - j11_dtest_jars_build - start_j11_dtests: type: approval - j11_dtests: @@ -9481,6 +10438,12 @@ workflows: requires: - start_j17_cqlsh_tests_offheap - j11_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - start_j11_upgrade_dtests + - j11_build java11_pre-commit_tests: jobs: - start_pre-commit_tests: @@ -9596,6 +10559,15 @@ workflows: requires: - start_utests_system_keyspace_directory - j11_build + - start_jvm_upgrade_dtests: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_jvm_upgrade_dtests + - j11_jvm_upgrade_dtests: + requires: + - j11_dtest_jars_build - j11_dtests: requires: - j11_build @@ -9704,6 +10676,12 @@ workflows: requires: - start_j17_cqlsh-dtests-offheap - j11_build + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - j11_build + - start_j11_upgrade_dtests java17_separate_tests: jobs: - start_j17_build: diff --git a/.circleci/config_template.yml b/.circleci/config_template.yml index 931b224c5b..2b603d233f 100644 --- a/.circleci/config_template.yml +++ b/.circleci/config_template.yml @@ -224,14 +224,14 @@ j8_repeated_dtest_executor: &j8_repeated_dtest_executor name: java8-executor parallelism: 4 -j8_repeated_upgrade_dtest_executor: &j8_repeated_upgrade_dtest_executor +j11_repeated_upgrade_dtest_executor: &j11_repeated_upgrade_dtest_executor executor: - name: java8-executor + name: java11-executor parallelism: 4 -j8_repeated_jvm_upgrade_dtest_executor: &j8_repeated_jvm_upgrade_dtest_executor +j11_repeated_jvm_upgrade_dtest_executor: &j11_repeated_jvm_upgrade_dtest_executor executor: - name: java8-executor + name: java11-executor parallelism: 4 j11_repeated_utest_executor: &j11_repeated_utest_executor @@ -550,24 +550,6 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j11_utests_system_keyspace_directory_repeat - j8_build - - start_j8_dtest_jars_build: - type: approval - - j8_dtest_jars_build: - requires: - - j8_build - - start_j8_dtest_jars_build - - start_jvm_upgrade_dtests: - type: approval - - j8_jvm_upgrade_dtests: - requires: - - start_jvm_upgrade_dtests - - j8_dtest_jars_build - - start_jvm_upgrade_dtests_repeat: - type: approval - - j8_jvm_upgrade_dtests_repeat: - requires: - - start_jvm_upgrade_dtests_repeat - - j8_dtest_jars_build # Python DTests - start_j8_dtests: type: approval @@ -669,13 +651,6 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j11_dtests_large_vnode_repeat - j8_build - # Java 8 upgrade tests - - start_upgrade_dtests: - type: approval - - j8_upgrade_dtests: - requires: - - start_upgrade_dtests - - j8_build # Java 8 cqlsh dtests - start_j8_cqlsh_tests: type: approval @@ -800,13 +775,6 @@ j8_separate_jobs: &j8_separate_jobs requires: - start_j11_dtests_vnode_repeat - j8_build - # Repeated Python upgrade dtest - - start_j8_upgrade_dtests_repeat: - type: approval - - j8_upgrade_dtests_repeat: - requires: - - start_j8_upgrade_dtests_repeat - - j8_build j8_pre-commit_jobs: &j8_pre-commit_jobs jobs: @@ -1008,18 +976,6 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs requires: - start_utests_system_keyspace_directory - j8_build - - start_jvm_upgrade_dtests: - type: approval - - j8_dtest_jars_build: - requires: - - j8_build - - start_jvm_upgrade_dtests - - j8_jvm_upgrade_dtests: - requires: - - j8_dtest_jars_build - - j8_jvm_upgrade_dtests_repeat: - requires: - - j8_dtest_jars_build # Python DTests - j8_dtests: requires: @@ -1103,17 +1059,6 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs requires: - start_j11_dtests_large - j8_build - # Java 8 upgrade tests (on request) - - start_upgrade_dtests: - type: approval - - j8_upgrade_dtests: - requires: - - j8_build - - start_upgrade_dtests - - j8_upgrade_dtests_repeat: - requires: - - j8_build - - start_upgrade_dtests # Java 8 cqlsh dtests - j8_cqlsh_dtests_py3: requires: @@ -1435,6 +1380,32 @@ j11_separate_jobs: &j11_separate_jobs requires: - start_j11_utests_system_keyspace_directory_repeat - j11_build + # Java upgrade tests + - start_j11_dtest_jars_build: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_j11_dtest_jars_build + - start_jvm_upgrade_dtests: + type: approval + - j11_jvm_upgrade_dtests: + requires: + - start_jvm_upgrade_dtests + - j11_dtest_jars_build + - start_jvm_upgrade_dtests_repeat: + type: approval + - j11_jvm_upgrade_dtests_repeat: + requires: + - start_jvm_upgrade_dtests_repeat + - j11_dtest_jars_build + # Java 11 upgrade tests + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - start_j11_upgrade_dtests + - j11_build # Java 11 repeated utest - start_j11_repeated_ant_test: type: approval @@ -1455,6 +1426,13 @@ j11_separate_jobs: &j11_separate_jobs requires: - start_j11_dtests_vnode_repeat - j11_build + # Repeated Python upgrade dtest + - start_j11_upgrade_dtests_repeat: + type: approval + - j11_upgrade_dtests_repeat: + requires: + - start_j11_upgrade_dtests_repeat + - j11_build j11_pre-commit_jobs: &j11_pre-commit_jobs jobs: @@ -1644,6 +1622,30 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs requires: - start_utests_system_keyspace_directory - j11_build + # Java upgrade tests + - start_jvm_upgrade_dtests: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_jvm_upgrade_dtests + - j11_jvm_upgrade_dtests: + requires: + - j11_dtest_jars_build + - j11_jvm_upgrade_dtests_repeat: + requires: + - j11_dtest_jars_build + # Java 11 upgrade tests (on request) + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - j11_build + - start_j11_upgrade_dtests + - j11_upgrade_dtests_repeat: + requires: + - j11_build + - start_j11_upgrade_dtests workflows: version: 2 @@ -1706,8 +1708,8 @@ jobs: executor: java11-executor <<: *build_common - j8_dtest_jars_build: - executor: java8-executor + j11_dtest_jars_build: + executor: java11-executor parallelism: 1 steps: - attach_workspace: @@ -1809,8 +1811,8 @@ jobs: target: "testclasslist" arguments: "-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'" - j8_jvm_upgrade_dtests: - <<: *j8_medium_par_executor + j11_jvm_upgrade_dtests: + <<: *j11_medium_par_executor steps: - attach_workspace: at: /home/cassandra @@ -2084,18 +2086,18 @@ jobs: file_tag: j11_without_vnodes pytest_extra_args: '--skip-resource-intensive-tests' - j8_upgrade_dtests: - <<: *j8_par_executor + j11_upgrade_dtests: + <<: *j11_par_executor steps: - attach_workspace: at: /home/cassandra - clone_dtest - create_venv - create_dtest_containers: - file_tag: j8_upgradetests_without_vnodes + file_tag: j11_upgradetests_without_vnodes run_dtests_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' - run_dtests: - file_tag: j8_upgradetests_without_vnodes + file_tag: j11_upgradetests_without_vnodes pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' j8_cqlsh_dtests_py3_vnode: @@ -2636,8 +2638,8 @@ jobs: - log_environment - run_simulator_dtests_repeat - j8_jvm_upgrade_dtests_repeat: - <<: *j8_repeated_jvm_upgrade_dtest_executor + j11_jvm_upgrade_dtests_repeat: + <<: *j11_repeated_jvm_upgrade_dtest_executor steps: - attach_workspace: at: /home/cassandra @@ -2836,8 +2838,8 @@ jobs: stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests" - j8_upgrade_dtests_repeat: - <<: *j8_repeated_upgrade_dtest_executor + j11_upgrade_dtests_repeat: + <<: *j11_repeated_upgrade_dtest_executor steps: - attach_workspace: at: /home/cassandra @@ -2921,7 +2923,7 @@ commands: cd ~/cassandra mkdir ~/dtest_jars git remote add apache https://github.com/apache/cassandra.git - for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 cassandra-4.1 trunk; do + for branch in cassandra-4.0 cassandra-4.1 trunk; do # check out the correct cassandra version: git remote set-branches --add apache '$branch' git fetch --depth 1 apache $branch diff --git a/.circleci/config_template.yml.PAID.patch b/.circleci/config_template.yml.PAID.patch index b80b4f1dce..ce9012dd25 100644 --- a/.circleci/config_template.yml.PAID.patch +++ b/.circleci/config_template.yml.PAID.patch @@ -1,6 +1,6 @@ ---- a/config_template.yml -+++ b/config_template.yml.PAID -@@ -157,14 +157,14 @@ default_env_vars: &default_env_vars +--- config_template.yml 2023-06-06 17:25:41.000000000 -0400 ++++ config_template.yml.PAID 2023-06-06 17:28:54.000000000 -0400 +@@ -157,14 +157,14 @@ j8_par_executor: &j8_par_executor executor: name: java8-executor @@ -19,7 +19,7 @@ j8_small_executor: &j8_small_executor executor: -@@ -172,29 +172,41 @@ j8_small_executor: &j8_small_executor +@@ -172,29 +172,41 @@ exec_resource_class: medium parallelism: 1 @@ -29,9 +29,9 @@ + exec_resource_class: large + parallelism: 50 + -+j8_very_large_par_executor: &j8_very_large_par_executor ++j11_very_large_par_executor: &j11_very_large_par_executor + executor: -+ name: java8-executor ++ name: java11-executor + exec_resource_class: large + parallelism: 100 + @@ -68,7 +68,7 @@ j11_small_executor: &j11_small_executor executor: -@@ -205,44 +217,56 @@ j11_small_executor: &j11_small_executor +@@ -205,44 +217,56 @@ j11_medium_par_executor: &j11_medium_par_executor executor: name: java11-executor @@ -104,16 +104,16 @@ + exec_resource_class: large + parallelism: 25 - j8_repeated_upgrade_dtest_executor: &j8_repeated_upgrade_dtest_executor + j11_repeated_upgrade_dtest_executor: &j11_repeated_upgrade_dtest_executor executor: - name: java8-executor + name: java11-executor - parallelism: 4 + exec_resource_class: xlarge + parallelism: 25 - j8_repeated_jvm_upgrade_dtest_executor: &j8_repeated_jvm_upgrade_dtest_executor + j11_repeated_jvm_upgrade_dtest_executor: &j11_repeated_jvm_upgrade_dtest_executor executor: - name: java8-executor + name: java11-executor - parallelism: 4 + exec_resource_class: large + parallelism: 25 @@ -134,7 +134,7 @@ j8_separate_jobs: &j8_separate_jobs jobs: -@@ -1998,7 +2022,7 @@ jobs: +@@ -2000,7 +2024,7 @@ target: testclasslist-system-keyspace-directory j8_dtests_vnode: @@ -143,7 +143,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2012,7 +2036,7 @@ jobs: +@@ -2014,7 +2038,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' j8_dtests_offheap: @@ -152,7 +152,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2026,7 +2050,7 @@ jobs: +@@ -2028,7 +2052,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' j11_dtests_vnode: @@ -161,7 +161,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2041,7 +2065,7 @@ jobs: +@@ -2043,7 +2067,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' j11_dtests_offheap: @@ -170,7 +170,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2056,7 +2080,7 @@ jobs: +@@ -2058,7 +2082,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' j8_dtests: @@ -179,7 +179,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2070,7 +2094,7 @@ jobs: +@@ -2072,7 +2096,7 @@ pytest_extra_args: '--skip-resource-intensive-tests' j11_dtests: @@ -188,16 +188,16 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2085,7 +2109,7 @@ jobs: +@@ -2087,7 +2111,7 @@ pytest_extra_args: '--skip-resource-intensive-tests' - j8_upgrade_dtests: -- <<: *j8_par_executor -+ <<: *j8_very_large_par_executor + j11_upgrade_dtests: +- <<: *j11_par_executor ++ <<: *j11_very_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -2099,7 +2123,7 @@ jobs: +@@ -2101,7 +2125,7 @@ pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' j8_cqlsh_dtests_py3_vnode: @@ -206,7 +206,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2114,7 +2138,7 @@ jobs: +@@ -2116,7 +2140,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j8_cqlsh_dtests_py3_offheap: @@ -215,7 +215,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2129,7 +2153,7 @@ jobs: +@@ -2131,7 +2155,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j8_cqlsh_dtests_py38_vnode: @@ -224,7 +224,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2147,7 +2171,7 @@ jobs: +@@ -2149,7 +2173,7 @@ python_version: '3.8' j8_cqlsh_dtests_py311_vnode: @@ -233,7 +233,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2165,7 +2189,7 @@ jobs: +@@ -2167,7 +2191,7 @@ python_version: '3.11' j8_cqlsh_dtests_py38_offheap: @@ -242,7 +242,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2183,7 +2207,7 @@ jobs: +@@ -2185,7 +2209,7 @@ python_version: '3.8' j8_cqlsh_dtests_py311_offheap: @@ -251,7 +251,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2201,7 +2225,7 @@ jobs: +@@ -2203,7 +2227,7 @@ python_version: '3.11' j8_cqlsh_dtests_py3: @@ -260,7 +260,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2216,7 +2240,7 @@ jobs: +@@ -2218,7 +2242,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j8_cqlsh_dtests_py38: @@ -269,7 +269,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2234,7 +2258,7 @@ jobs: +@@ -2236,7 +2260,7 @@ python_version: '3.8' j8_cqlsh_dtests_py311: @@ -278,7 +278,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2252,7 +2276,7 @@ jobs: +@@ -2254,7 +2278,7 @@ python_version: '3.11' j11_cqlsh_dtests_py3_vnode: @@ -287,7 +287,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2267,7 +2291,7 @@ jobs: +@@ -2269,7 +2293,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py3_offheap: @@ -296,7 +296,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2282,7 +2306,7 @@ jobs: +@@ -2284,7 +2308,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py38_vnode: @@ -305,7 +305,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2300,7 +2324,7 @@ jobs: +@@ -2302,7 +2326,7 @@ python_version: '3.8' j11_cqlsh_dtests_py311_vnode: @@ -314,7 +314,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2318,7 +2342,7 @@ jobs: +@@ -2320,7 +2344,7 @@ python_version: '3.11' j11_cqlsh_dtests_py38_offheap: @@ -323,7 +323,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2336,7 +2360,7 @@ jobs: +@@ -2338,7 +2362,7 @@ python_version: '3.8' j11_cqlsh_dtests_py311_offheap: @@ -332,7 +332,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2354,7 +2378,7 @@ jobs: +@@ -2356,7 +2380,7 @@ python_version: '3.11' j11_cqlsh_dtests_py3: @@ -341,7 +341,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2369,7 +2393,7 @@ jobs: +@@ -2371,7 +2395,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py38: @@ -350,7 +350,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2387,7 +2411,7 @@ jobs: +@@ -2389,7 +2413,7 @@ python_version: '3.8' j11_cqlsh_dtests_py311: @@ -359,7 +359,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2405,7 +2429,7 @@ jobs: +@@ -2407,7 +2431,7 @@ python_version: '3.11' j8_dtests_large_vnode: @@ -368,7 +368,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2419,7 +2443,7 @@ jobs: +@@ -2421,7 +2445,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests' j8_dtests_large: @@ -377,7 +377,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2433,7 +2457,7 @@ jobs: +@@ -2435,7 +2459,7 @@ pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests' j11_dtests_large_vnode: @@ -386,7 +386,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2447,7 +2471,7 @@ jobs: +@@ -2449,7 +2473,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests' j11_dtests_large: diff --git a/.circleci/config_template_11_and_17.yml b/.circleci/config_template_11_and_17.yml index dc662ca8f1..f74e89298e 100644 --- a/.circleci/config_template_11_and_17.yml +++ b/.circleci/config_template_11_and_17.yml @@ -95,6 +95,14 @@ default_env_vars: &default_env_vars # The number of times that new, modified or manually specified JVM dtests should be run. REPEATED_JVM_DTESTS_COUNT: 500 + # Comma-separated list of tests that should be included in the repeated run for JVM upgrade dtests, + # in addition to automatically detected new and modified tests. For example: + # REPEATED_JVM_UPGRADE_DTESTS: org.apache.cassandra.distributed.upgrade.GroupByTest + # REPEATED_JVM_UPGRADE_DTESTS: org.apache.cassandra.distributed.upgrade.GroupByTest#testReads + REPEATED_JVM_UPGRADE_DTESTS: + # The number of times that new, modified or manually specified JVM upgrade dtests should be run. + REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + # Comma-separated list of Python dtests that should be repeatedly run, for example: # REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py # REPEATED_DTESTS: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke @@ -113,6 +121,14 @@ default_env_vars: &default_env_vars # The number of times that the manually specified Python large dtests should be run. REPEATED_LARGE_DTESTS_COUNT: 100 + # Comma-separated list of Python dtests that should be repeatedly run, for example: + # REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py + # REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py::TestCQLNodes2RF1_Upgrade_current_4_0_x_To_indev_4_0_x + # REPEATED_UPGRADE_DTESTS: upgrade_tests/cql_tests.py,upgrade_tests/paging_test.py + REPEATED_UPGRADE_DTESTS: + # The number of times that the manually specified Python upgrade dtests should be run. + REPEATED_UPGRADE_DTESTS_COUNT: 25 + # The Ant test target to run, for example: # REPEATED_ANT_TEST_TARGET: testsome # REPEATED_ANT_TEST_TARGET: test-jvm-dtest-some @@ -199,6 +215,11 @@ j17_seq_executor: &j17_seq_executor #exec_resource_class: xlarge parallelism: 1 # sequential, single container tests: no parallelism benefits +j11_repeated_jvm_upgrade_dtest_executor: &j11_repeated_jvm_upgrade_dtest_executor + executor: + name: java11-executor + parallelism: 4 + j11_repeated_utest_executor: &j11_repeated_utest_executor executor: name: java11-executor @@ -219,6 +240,11 @@ j17_repeated_dtest_executor: &j17_repeated_dtest_executor name: java17-executor parallelism: 4 +j11_repeated_upgrade_dtest_executor: &j11_repeated_upgrade_dtest_executor + executor: + name: java11-executor + parallelism: 4 + j11_separate_jobs: &j11_separate_jobs jobs: - start_j11_build: @@ -525,6 +551,25 @@ j11_separate_jobs: &j11_separate_jobs requires: - start_j17_utests_system_keyspace_directory_repeat - j11_build + # Java upgrade tests + - start_j11_dtest_jars_build: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_j11_dtest_jars_build + - start_jvm_upgrade_dtests: + type: approval + - j11_jvm_upgrade_dtests: + requires: + - start_jvm_upgrade_dtests + - j11_dtest_jars_build + - start_jvm_upgrade_dtests_repeat: + type: approval + - j11_jvm_upgrade_dtests_repeat: + requires: + - start_jvm_upgrade_dtests_repeat + - j11_dtest_jars_build # Python DTests - start_j11_dtests: type: approval @@ -710,6 +755,13 @@ j11_separate_jobs: &j11_separate_jobs requires: - start_j17_cqlsh_tests_offheap - j11_build + # Java 11 upgrade tests + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - start_j11_upgrade_dtests + - j11_build # Java 11 repeated utest - start_j11_repeated_ant_test: type: approval @@ -750,6 +802,13 @@ j11_separate_jobs: &j11_separate_jobs requires: - start_j17_dtests_vnode_repeat - j11_build + # Repeated Python upgrade dtest + - start_j11_upgrade_dtests_repeat: + type: approval + - j11_upgrade_dtests_repeat: + requires: + - start_j11_upgrade_dtests_repeat + - j11_build j11_pre-commit_jobs: &j11_pre-commit_jobs jobs: @@ -951,6 +1010,19 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs requires: - start_utests_system_keyspace_directory - j11_build + # Java upgrade tests + - start_jvm_upgrade_dtests: + type: approval + - j11_dtest_jars_build: + requires: + - j11_build + - start_jvm_upgrade_dtests + - j11_jvm_upgrade_dtests: + requires: + - j11_dtest_jars_build + - j11_jvm_upgrade_dtests_repeat: + requires: + - j11_dtest_jars_build # Python DTests - j11_dtests: requires: @@ -1102,6 +1174,17 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs requires: - start_j17_cqlsh-dtests-offheap - j11_build + # Java 11 upgrade tests (on request) + - start_j11_upgrade_dtests: + type: approval + - j11_upgrade_dtests: + requires: + - j11_build + - start_j11_upgrade_dtests + - j11_upgrade_dtests_repeat: + requires: + - j11_build + - start_j11_upgrade_dtests j17_separate_jobs: &j17_separate_jobs jobs: @@ -1569,6 +1652,7 @@ executors: <<: *default_env_vars JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + CASSANDRA_USE_JDK11: true java17-executor: parameters: @@ -1687,6 +1771,19 @@ jobs: classlistprefix: distributed target: "testclasslist" + j11_jvm_upgrade_dtests: + <<: *j11_medium_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - create_junit_containers: + classlistprefix: distributed + extra_filters: "| grep upgrade" + - log_environment + - run_parallel_junit_tests: + classlistprefix: distributed + target: "testclasslist" + j17_jvm_dtests_vnode: <<: *j17_small_par_executor steps: @@ -1963,6 +2060,20 @@ jobs: file_tag: j17_without_vnodes pytest_extra_args: '--skip-resource-intensive-tests' + j11_upgrade_dtests: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j11_upgradetests_without_vnodes + run_dtests_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' + - run_dtests: + file_tag: j11_upgradetests_without_vnodes + pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' + j11_cqlsh_dtests_py3_vnode: <<: *j11_par_executor steps: @@ -2469,6 +2580,14 @@ jobs: - log_environment - run_utests_stress_repeat + j11_jvm_upgrade_dtests_repeat: + <<: *j11_repeated_jvm_upgrade_dtest_executor + steps: + - attach_workspace: + at: /home/cassandra + - log_environment + - run_jvm_upgrade_dtests_repeat + j11_jvm_dtests_repeat: <<: *j11_repeated_utest_executor steps: @@ -2685,6 +2804,20 @@ jobs: stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} extra_dtest_args: "--only-resource-intensive-tests --force-resource-intensive-tests" + j11_upgrade_dtests_repeat: + <<: *j11_repeated_upgrade_dtest_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - run_repeated_dtest: + tests: ${REPEATED_UPGRADE_DTESTS} + vnodes: "false" + upgrade: "true" + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} + count: ${REPEATED_UPGRADE_DTESTS_COUNT} + commands: log_environment: steps: @@ -2756,7 +2889,7 @@ commands: cd ~/cassandra mkdir ~/dtest_jars git remote add apache https://github.com/apache/cassandra.git - for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 cassandra-4.1 trunk; do + for branch in cassandra-4.0 cassandra-4.1 trunk; do # check out the correct cassandra version: git remote set-branches --add apache '$branch' git fetch --depth 1 apache $branch @@ -3181,6 +3314,15 @@ commands: vnodes: true stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} + run_jvm_upgrade_dtests_repeat: + steps: + - run_repeated_utests: + target: test-jvm-dtest-some + tests: ${REPEATED_JVM_UPGRADE_DTESTS} + count: ${REPEATED_JVM_UPGRADE_DTESTS_COUNT} + vnodes: false + stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE} + run_repeated_utests: parameters: target: diff --git a/.circleci/config_template_11_and_17.yml.PAID.patch b/.circleci/config_template_11_and_17.yml.PAID.patch index 2f55409348..e6943562c2 100644 --- a/.circleci/config_template_11_and_17.yml.PAID.patch +++ b/.circleci/config_template_11_and_17.yml.PAID.patch @@ -1,6 +1,6 @@ ---- a/config_template_11_and_17.yml -+++ b/config_template_11_and_17.yml.PAID -@@ -141,14 +141,14 @@ default_env_vars: &default_env_vars +--- config_template_11_and_17.yml 2023-06-09 19:10:52.000000000 -0400 ++++ config_template_11_and_17.yml.PAID 2023-06-09 19:34:32.000000000 -0400 +@@ -157,14 +157,14 @@ j11_par_executor: &j11_par_executor executor: name: java11-executor @@ -19,7 +19,7 @@ j11_small_executor: &j11_small_executor executor: -@@ -156,30 +156,41 @@ j11_small_executor: &j11_small_executor +@@ -172,30 +172,41 @@ exec_resource_class: medium parallelism: 1 @@ -68,7 +68,7 @@ j17_small_executor: &j17_small_executor executor: -@@ -190,34 +201,44 @@ j17_small_executor: &j17_small_executor +@@ -206,44 +217,56 @@ j17_medium_par_executor: &j17_medium_par_executor executor: name: java17-executor @@ -90,6 +90,13 @@ + exec_resource_class: medium parallelism: 1 # sequential, single container tests: no parallelism benefits + j11_repeated_jvm_upgrade_dtest_executor: &j11_repeated_jvm_upgrade_dtest_executor + executor: + name: java11-executor +- parallelism: 4 ++ exec_resource_class: large ++ parallelism: 25 + j11_repeated_utest_executor: &j11_repeated_utest_executor executor: name: java11-executor @@ -116,11 +123,18 @@ name: java17-executor - parallelism: 4 + exec_resource_class: large ++ parallelism: 25 + + j11_repeated_upgrade_dtest_executor: &j11_repeated_upgrade_dtest_executor + executor: + name: java11-executor +- parallelism: 4 ++ exec_resource_class: xlarge + parallelism: 25 j11_separate_jobs: &j11_separate_jobs jobs: -@@ -1877,7 +1898,7 @@ jobs: +@@ -1974,7 +1997,7 @@ target: testclasslist-system-keyspace-directory j11_dtests_vnode: @@ -129,7 +143,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1891,7 +1912,7 @@ jobs: +@@ -1988,7 +2011,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' j11_dtests_offheap: @@ -138,7 +152,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1905,7 +1926,7 @@ jobs: +@@ -2002,7 +2025,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' j17_dtests_vnode: @@ -147,7 +161,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1920,7 +1941,7 @@ jobs: +@@ -2017,7 +2040,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests' j17_dtests_offheap: @@ -156,7 +170,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1935,7 +1956,7 @@ jobs: +@@ -2032,7 +2055,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests' j11_dtests: @@ -165,7 +179,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1949,7 +1970,7 @@ jobs: +@@ -2046,7 +2069,7 @@ pytest_extra_args: '--skip-resource-intensive-tests' j17_dtests: @@ -174,16 +188,25 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1964,7 +1985,7 @@ jobs: +@@ -2061,7 +2084,7 @@ pytest_extra_args: '--skip-resource-intensive-tests' + j11_upgrade_dtests: +- <<: *j11_par_executor ++ <<: *j11_very_large_par_executor + steps: + - attach_workspace: + at: /home/cassandra +@@ -2075,7 +2098,7 @@ + pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all' + j11_cqlsh_dtests_py3_vnode: - <<: *j11_par_executor + <<: *j11_large_par_executor steps: - attach_workspace: at: /home/cassandra -@@ -1979,7 +2000,7 @@ jobs: +@@ -2090,7 +2113,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py3_offheap: @@ -192,7 +215,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -1994,7 +2015,7 @@ jobs: +@@ -2105,7 +2128,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py38_vnode: @@ -201,7 +224,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2012,7 +2033,7 @@ jobs: +@@ -2123,7 +2146,7 @@ python_version: '3.8' j11_cqlsh_dtests_py311_vnode: @@ -210,7 +233,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2030,7 +2051,7 @@ jobs: +@@ -2141,7 +2164,7 @@ python_version: '3.11' j11_cqlsh_dtests_py38_offheap: @@ -219,7 +242,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2048,7 +2069,7 @@ jobs: +@@ -2159,7 +2182,7 @@ python_version: '3.8' j11_cqlsh_dtests_py311_offheap: @@ -228,7 +251,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2066,7 +2087,7 @@ jobs: +@@ -2177,7 +2200,7 @@ python_version: '3.11' j11_cqlsh_dtests_py3: @@ -237,7 +260,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2081,7 +2102,7 @@ jobs: +@@ -2192,7 +2215,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j11_cqlsh_dtests_py38: @@ -246,7 +269,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2099,7 +2120,7 @@ jobs: +@@ -2210,7 +2233,7 @@ python_version: '3.8' j11_cqlsh_dtests_py311: @@ -255,7 +278,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2117,7 +2138,7 @@ jobs: +@@ -2228,7 +2251,7 @@ python_version: '3.11' j17_cqlsh_dtests_py3_vnode: @@ -264,7 +287,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2132,7 +2153,7 @@ jobs: +@@ -2243,7 +2266,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j17_cqlsh_dtests_py3_offheap: @@ -273,7 +296,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2147,7 +2168,7 @@ jobs: +@@ -2258,7 +2281,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j17_cqlsh_dtests_py38_vnode: @@ -282,7 +305,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2165,7 +2186,7 @@ jobs: +@@ -2276,7 +2299,7 @@ python_version: '3.8' j17_cqlsh_dtests_py311_vnode: @@ -291,7 +314,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2183,7 +2204,7 @@ jobs: +@@ -2294,7 +2317,7 @@ python_version: '3.11' j17_cqlsh_dtests_py38_offheap: @@ -300,7 +323,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2201,7 +2222,7 @@ jobs: +@@ -2312,7 +2335,7 @@ python_version: '3.8' j17_cqlsh_dtests_py311_offheap: @@ -309,7 +332,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2219,7 +2240,7 @@ jobs: +@@ -2330,7 +2353,7 @@ python_version: '3.11' j17_cqlsh_dtests_py3: @@ -318,7 +341,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2234,7 +2255,7 @@ jobs: +@@ -2345,7 +2368,7 @@ extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' j17_cqlsh_dtests_py38: @@ -327,7 +350,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2252,7 +2273,7 @@ jobs: +@@ -2363,7 +2386,7 @@ python_version: '3.8' j17_cqlsh_dtests_py311: @@ -336,7 +359,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2270,7 +2291,7 @@ jobs: +@@ -2381,7 +2404,7 @@ python_version: '3.11' j11_dtests_large_vnode: @@ -345,7 +368,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2284,7 +2305,7 @@ jobs: +@@ -2395,7 +2418,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests' j11_dtests_large: @@ -354,7 +377,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2298,7 +2319,7 @@ jobs: +@@ -2409,7 +2432,7 @@ pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests' j17_dtests_large_vnode: @@ -363,7 +386,7 @@ steps: - attach_workspace: at: /home/cassandra -@@ -2312,7 +2333,7 @@ jobs: +@@ -2423,7 +2446,7 @@ pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests' j17_dtests_large: diff --git a/.circleci/generate.sh b/.circleci/generate.sh index 1cd8fa09ea..5178e11d7f 100755 --- a/.circleci/generate.sh +++ b/.circleci/generate.sh @@ -308,7 +308,7 @@ delete_repeated_jobs() fi if (! (echo "$env_vars" | grep -q "REPEATED_JVM_UPGRADE_DTESTS=")); then delete_job "$1" "start_jvm_upgrade_dtests_repeat" - delete_job "$1" "j8_jvm_upgrade_dtests_repeat" + delete_job "$1" "j11_jvm_upgrade_dtests_repeat" fi if (! (echo "$env_vars" | grep -q "REPEATED_DTESTS=")); then delete_job "$1" "j8_dtests_repeat" @@ -325,7 +325,7 @@ delete_repeated_jobs() delete_job "$1" "j11_dtests_large_vnode_repeat" fi if (! (echo "$env_vars" | grep -q "REPEATED_UPGRADE_DTESTS=")); then - delete_job "$1" "j8_upgrade_dtests_repeat" + delete_job "$1" "j11_upgrade_dtests_repeat" fi if (! (echo "$env_vars" | grep -q "REPEATED_ANT_TEST_CLASS=")); then delete_job "$1" "j8_repeated_ant_test" diff --git a/.circleci/generate_11_and_17.sh b/.circleci/generate_11_and_17.sh index 4f17017e8d..8e031e9b76 100755 --- a/.circleci/generate_11_and_17.sh +++ b/.circleci/generate_11_and_17.sh @@ -57,10 +57,14 @@ print_help() echo " -e REPEATED_SIMULATOR_DTESTS_COUNT=500" echo " -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest" echo " -e REPEATED_JVM_DTESTS_COUNT=500" + echo " -e REPEATED_JVM_UPGRADE_DTESTS=org.apache.cassandra.distributed.upgrade.GroupByTest" + echo " -e REPEATED_JVM_UPGRADE_DTESTS_COUNT=500" echo " -e REPEATED_DTESTS=cdc_test.py cqlsh_tests/test_cqlsh.py::TestCqlshSmoke" echo " -e REPEATED_DTESTS_COUNT=500" echo " -e REPEATED_LARGE_DTESTS=replace_address_test.py::TestReplaceAddress::test_replace_stopped_node" echo " -e REPEATED_LARGE_DTESTS=100" + echo " -e REPEATED_UPGRADE_DTESTS=upgrade_tests/cql_tests.py upgrade_tests/paging_test.py" + echo " -e REPEATED_UPGRADE_DTESTS_COUNT=25" echo " -e REPEATED_ANT_TEST_TARGET=testsome" echo " -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest" echo " -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable" @@ -130,10 +134,14 @@ if $has_env_vars && $check_env_vars; then [ "$key" != "REPEATED_SIMULATOR_DTESTS_COUNT" ] && [ "$key" != "REPEATED_JVM_DTESTS" ] && [ "$key" != "REPEATED_JVM_DTESTS_COUNT" ] && + [ "$key" != "REPEATED_JVM_UPGRADE_DTESTS" ] && + [ "$key" != "REPEATED_JVM_UPGRADE_DTESTS_COUNT" ] && [ "$key" != "REPEATED_DTESTS" ] && [ "$key" != "REPEATED_DTESTS_COUNT" ] && [ "$key" != "REPEATED_LARGE_DTESTS" ] && [ "$key" != "REPEATED_LARGE_DTESTS_COUNT" ] && + [ "$key" != "REPEATED_UPGRADE_DTESTS" ] && + [ "$key" != "REPEATED_UPGRADE_DTESTS_COUNT" ] && [ "$key" != "REPEATED_ANT_TEST_TARGET" ] && [ "$key" != "REPEATED_ANT_TEST_CLASS" ] && [ "$key" != "REPEATED_ANT_TEST_METHODS" ] && @@ -228,6 +236,7 @@ if $detect_changed_tests; then add_diff_tests "REPEATED_UTESTS_FQLTOOL" "tools/fqltool/test/unit/" "org.apache.cassandra.fqltool" add_diff_tests "REPEATED_SIMULATOR_DTESTS" "test/simulator/test/" "org.apache.cassandra.simulator.test" add_diff_tests "REPEATED_JVM_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.test" + add_diff_tests "REPEATED_JVM_UPGRADE_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.upgrade" fi # replace environment variables @@ -299,6 +308,10 @@ delete_repeated_jobs() delete_job "$1" "j17_jvm_dtests_repeat" delete_job "$1" "j17_jvm_dtests_vnode_repeat" fi + if (! (echo "$env_vars" | grep -q "REPEATED_JVM_UPGRADE_DTESTS=")); then + delete_job "$1" "start_jvm_upgrade_dtests_repeat" + delete_job "$1" "j11_jvm_upgrade_dtests_repeat" + fi if (! (echo "$env_vars" | grep -q "REPEATED_DTESTS=")); then delete_job "$1" "j11_dtests_repeat" delete_job "$1" "j11_dtests_vnode_repeat" @@ -313,6 +326,9 @@ delete_repeated_jobs() delete_job "$1" "j17_dtests_large_repeat" delete_job "$1" "j17_dtests_large_vnode_repeat" fi + if (! (echo "$env_vars" | grep -q "REPEATED_UPGRADE_DTESTS=")); then + delete_job "$1" "j11_upgrade_dtests_repeat" + fi if (! (echo "$env_vars" | grep -q "REPEATED_ANT_TEST_CLASS=")); then delete_job "$1" "j11_repeated_ant_test" delete_job "$1" "j17_repeated_ant_test"