diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml index 5d8d88417f..b55b1fb0ca 100644 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@ -61,6 +61,16 @@ with_dtests_jobs: &with_dtest_jobs - utests_stress: requires: - start_utests_stress + - start_jvm_upgrade_dtest: + type: approval + requires: + - build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build # Java 8 dtests (on request) - start_j8_dtests: type: approval @@ -127,6 +137,18 @@ jobs: - cassandra - .m2 + dtest_jars_build: + executor: java8-executor + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - build_cassandra_dtest_jars + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + j8_unit_tests: <<: *j8_par_executor steps: @@ -144,6 +166,14 @@ jobs: - run_junit_tests: target: test-jvm-dtest-forking + j8_jvm_upgrade_dtests: + <<: *j8_seq_executor + steps: + - attach_workspace: + at: /home/cassandra + - run_junit_tests: + target: test-jvm-upgrade-dtest-forking + utests_long: <<: *j8_seq_executor steps: @@ -277,6 +307,47 @@ commands: fi no_output_timeout: 15m + build_cassandra_dtest_jars: + steps: + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + run_eclipse_warnings: steps: - run: @@ -325,6 +396,9 @@ commands: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean <> no_output_timeout: <> - store_test_results: diff --git a/.circleci/config.yml b/.circleci/config.yml index 7016d9bd9b..e0c65f41ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,51 @@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean test-jvm-upgrade-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -109,37 +155,11 @@ jobs: - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\ - \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -280,6 +300,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean stress-test no_output_timeout: 15m - store_test_results: @@ -419,36 +442,11 @@ jobs: - run: name: Determine Tests to Run (j8_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: name: Run dtests (j8_with_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\ - \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -490,6 +488,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean test-jvm-dtest-forking no_output_timeout: 15m - store_test_results: @@ -533,6 +534,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean long-test no_output_timeout: 15m - store_test_results: @@ -644,6 +648,75 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + dtest_jars_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 build_and_run_tests: @@ -676,6 +749,16 @@ workflows: - utests_stress: requires: - start_utests_stress + - start_jvm_upgrade_dtest: + type: approval + requires: + - build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build - start_j8_dtests: type: approval requires: @@ -693,468 +776,3 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests - -# Original config.yml file: -# version: 2.1 -# -# default_env_vars: &default_env_vars -# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# ANT_HOME: /usr/share/ant -# LANG: en_US.UTF-8 -# KEEP_TEST_DIR: true -# DEFAULT_DIR: /home/cassandra/cassandra-dtest -# PYTHONIOENCODING: utf-8 -# PYTHONUNBUFFERED: true -# CASS_DRIVER_NO_EXTENSIONS: true -# CASS_DRIVER_NO_CYTHON: true -# #Skip all syncing to disk to avoid performance issues in flaky CI environments -# CASSANDRA_SKIP_SYNC: true -# DTEST_REPO: git://github.com/apache/cassandra-dtest.git -# DTEST_BRANCH: master -# CCM_MAX_HEAP_SIZE: 1024M -# CCM_HEAP_NEWSIZE: 256M -# -# j8_par_executor: &j8_par_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 4 -# -# j8_seq_executor: &j8_seq_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 1 # sequential, single container tests: no parallelism benefits -# -# with_dtests_jobs: &with_dtest_jobs -# jobs: -# - build -# # Java 8 unit tests will be run automatically -# - j8_unit_tests: -# requires: -# - build -# - j8_jvm_dtests: -# requires: -# - build -# # specialized unit tests (all run on request using Java 8) -# - start_utests_long: -# type: approval -# requires: -# - build -# - utests_long: -# requires: -# - start_utests_long -# - start_utests_compression: -# type: approval -# requires: -# - build -# - utests_compression: -# requires: -# - start_utests_compression -# - start_utests_stress: -# type: approval -# requires: -# - build -# - utests_stress: -# requires: -# - start_utests_stress -# # Java 8 dtests (on request) -# - start_j8_dtests: -# type: approval -# requires: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - start_j8_dtests -# - j8_dtests-no-vnodes: -# requires: -# - start_j8_dtests -# # Java 8 upgrade tests -# - start_upgrade_tests: -# type: approval -# requires: -# - build -# - j8_upgradetests-no-vnodes: -# requires: -# - start_upgrade_tests -# -# with_dtest_jobs_only: &with_dtest_jobs_only -# jobs: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - build -# - j8_dtests-no-vnodes: -# requires: -# - build -# -# workflows: -# version: 2 -# build_and_run_tests: *with_dtest_jobs -# #build_and_run_tests: *with_dtest_jobs_only -# -# executors: -# java8-executor: -# parameters: -# exec_resource_class: -# type: string -# default: medium -# docker: -# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 -# resource_class: << parameters.exec_resource_class >> -# working_directory: ~/ -# shell: /bin/bash -eo pipefail -l -# environment: -# <<: *default_env_vars -# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# -# jobs: -# build: -# executor: java8-executor -# parallelism: 1 # This job doesn't benefit from parallelism -# steps: -# - log_environment -# - clone_cassandra -# - build_cassandra -# - run_eclipse_warnings -# - persist_to_workspace: -# root: /home/cassandra -# paths: -# - cassandra -# - .m2 -# -# j8_unit_tests: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests -# -# j8_jvm_dtests: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: test-jvm-dtest-forking -# -# utests_long: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: long-test -# -# utests_compression: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests: -# target: testclasslist-compression -# -# utests_stress: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: stress-test -# -# j8_dtests-with-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_with_vnodes -# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_with_vnodes -# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' -# -# j8_dtests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_without_vnodes -# run_dtests_extra_args: '--skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_without_vnodes -# pytest_extra_args: '--skip-resource-intensive-tests' -# -# j8_upgradetests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_upgradetests_without_vnodes -# run_dtests_extra_args: '--execute-upgrade-tests' -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# tests_filter_pattern: '^upgrade_tests' -# - run_dtests: -# file_tag: j8_upgradetests_without_vnodes -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# pytest_extra_args: '--execute-upgrade-tests' -# -# commands: -# log_environment: -# steps: -# - run: -# name: Log Environment Information -# command: | -# echo '*** id ***' -# id -# echo '*** cat /proc/cpuinfo ***' -# cat /proc/cpuinfo -# echo '*** free -m ***' -# free -m -# echo '*** df -m ***' -# df -m -# echo '*** ifconfig -a ***' -# ifconfig -a -# echo '*** uname -a ***' -# uname -a -# echo '*** mount ***' -# mount -# echo '*** env ***' -# env -# echo '*** java ***' -# which java -# java -version -# -# clone_cassandra: -# steps: -# - run: -# name: Clone Cassandra Repository (via git) -# command: | -# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra -# -# clone_dtest: -# steps: -# - run: -# name: Clone Cassandra dtest Repository (via git) -# command: | -# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest -# -# build_cassandra: -# steps: -# - run: -# name: Build Cassandra -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. -# for x in $(seq 1 3); do -# ${ANT_HOME}/bin/ant clean jar -# RETURN=\"$?\" -# if [ \"${RETURN}\" -eq \"0\" ]; then -# break -# fi -# done -# # Exit, if we didn't build successfully -# if [ \"${RETURN}\" -ne \"0\" ]; then -# echo \"Build failed with exit code: ${RETURN}\" -# exit ${RETURN} -# fi -# no_output_timeout: 15m -# -# run_eclipse_warnings: -# steps: -# - run: -# name: Run eclipse-warnings -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# ant eclipse-warnings -# -# create_junit_containers: -# steps: -# - run: -# name: Determine Unit Tests to Run -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# rm -fr ~/cassandra-dtest/upgrade_tests -# echo \"***java tests***\" -# -# # get all of our unit test filenames -# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt -# -# # split up the unit tests into groups based on the number of containers we have -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt -# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\" -# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# -# no_output_timeout: 15m -# -# run_junit_tests: -# parameters: -# target: -# type: string -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant clean <> -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# run_parallel_junit_tests: -# parameters: -# target: -# type: string -# default: testclasslist -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# create_venv: -# steps: -# - run: -# name: Configure virtualenv and python Dependencies -# command: | -# # note, this should be super quick as all dependencies should be pre-installed in the docker image -# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated -# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and -# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt -# pip3 freeze -# -# create_dtest_containers: -# parameters: -# file_tag: -# type: string -# run_dtests_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# tests_filter_pattern: -# type: string -# default: '' -# steps: -# - run: -# name: Determine Tests to Run (<>) -# no_output_timeout: 5m -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# cd cassandra-dtest -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# -# if [ -n '<>' ]; then -# export <> -# fi -# -# echo \"***Collected DTests (<>)***\" -# set -eo pipefail && ./run_dtests.py <> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<>_raw --cassandra-dir=../cassandra -# if [ -z '<>' ]; then -# mv /tmp/all_dtest_tests_<>_raw /tmp/all_dtest_tests_<> -# else -# grep -e '<>' /tmp/all_dtest_tests_<>_raw > /tmp/all_dtest_tests_<> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; } -# fi -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<> > /tmp/split_dtest_tests_<>.txt -# cat /tmp/split_dtest_tests_<>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<>_final.txt -# cat /tmp/split_dtest_tests_<>_final.txt -# -# run_dtests: -# parameters: -# file_tag: -# type: string -# pytest_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# steps: -# - run: -# name: Run dtests (<>) -# no_output_timeout: 15m -# command: | -# echo \"cat /tmp/split_dtest_tests_<>_final.txt\" -# cat /tmp/split_dtest_tests_<>_final.txt -# -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# if [ -n '<>' ]; then -# export <> -# fi -# -# java -version -# cd ~/cassandra-dtest -# mkdir -p /tmp/dtest -# -# echo \"env: $(env)\" -# echo \"** done env\" -# mkdir -p /tmp/results/dtests -# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee -# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<>_final.txt` -# set -o pipefail && cd ~/cassandra-dtest && pytest <> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt -# - store_test_results: -# path: /tmp/results -# - store_artifacts: -# path: /tmp/dtest -# destination: dtest_<> -# - store_artifacts: -# path: ~/cassandra-dtest/logs -# destination: dtest_<>_logs \ No newline at end of file diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES index de74d3319b..c6688d9d9f 100644 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@ -1,5 +1,51 @@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean test-jvm-upgrade-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -109,37 +155,11 @@ jobs: - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\ - \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -280,6 +300,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean stress-test no_output_timeout: 15m - store_test_results: @@ -419,36 +442,11 @@ jobs: - run: name: Determine Tests to Run (j8_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: name: Run dtests (j8_with_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\ - \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -490,6 +488,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean test-jvm-dtest-forking no_output_timeout: 15m - store_test_results: @@ -533,6 +534,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean long-test no_output_timeout: 15m - store_test_results: @@ -644,6 +648,75 @@ jobs: - CCM_HEAP_NEWSIZE: 512M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + dtest_jars_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 build_and_run_tests: @@ -676,6 +749,16 @@ workflows: - utests_stress: requires: - start_utests_stress + - start_jvm_upgrade_dtest: + type: approval + requires: + - build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build - start_j8_dtests: type: approval requires: @@ -693,468 +776,3 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests - -# Original config.yml file: -# version: 2.1 -# -# default_env_vars: &default_env_vars -# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# ANT_HOME: /usr/share/ant -# LANG: en_US.UTF-8 -# KEEP_TEST_DIR: true -# DEFAULT_DIR: /home/cassandra/cassandra-dtest -# PYTHONIOENCODING: utf-8 -# PYTHONUNBUFFERED: true -# CASS_DRIVER_NO_EXTENSIONS: true -# CASS_DRIVER_NO_CYTHON: true -# #Skip all syncing to disk to avoid performance issues in flaky CI environments -# CASSANDRA_SKIP_SYNC: true -# DTEST_REPO: git://github.com/apache/cassandra-dtest.git -# DTEST_BRANCH: master -# CCM_MAX_HEAP_SIZE: 2048M -# CCM_HEAP_NEWSIZE: 512M -# -# j8_par_executor: &j8_par_executor -# executor: -# name: java8-executor -# exec_resource_class: xlarge -# parallelism: 100 -# -# j8_seq_executor: &j8_seq_executor -# executor: -# name: java8-executor -# exec_resource_class: xlarge -# parallelism: 1 # sequential, single container tests: no parallelism benefits -# -# with_dtests_jobs: &with_dtest_jobs -# jobs: -# - build -# # Java 8 unit tests will be run automatically -# - j8_unit_tests: -# requires: -# - build -# - j8_jvm_dtests: -# requires: -# - build -# # specialized unit tests (all run on request using Java 8) -# - start_utests_long: -# type: approval -# requires: -# - build -# - utests_long: -# requires: -# - start_utests_long -# - start_utests_compression: -# type: approval -# requires: -# - build -# - utests_compression: -# requires: -# - start_utests_compression -# - start_utests_stress: -# type: approval -# requires: -# - build -# - utests_stress: -# requires: -# - start_utests_stress -# # Java 8 dtests (on request) -# - start_j8_dtests: -# type: approval -# requires: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - start_j8_dtests -# - j8_dtests-no-vnodes: -# requires: -# - start_j8_dtests -# # Java 8 upgrade tests -# - start_upgrade_tests: -# type: approval -# requires: -# - build -# - j8_upgradetests-no-vnodes: -# requires: -# - start_upgrade_tests -# -# with_dtest_jobs_only: &with_dtest_jobs_only -# jobs: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - build -# - j8_dtests-no-vnodes: -# requires: -# - build -# -# workflows: -# version: 2 -# build_and_run_tests: *with_dtest_jobs -# #build_and_run_tests: *with_dtest_jobs_only -# -# executors: -# java8-executor: -# parameters: -# exec_resource_class: -# type: string -# default: medium -# docker: -# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 -# resource_class: << parameters.exec_resource_class >> -# working_directory: ~/ -# shell: /bin/bash -eo pipefail -l -# environment: -# <<: *default_env_vars -# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# -# jobs: -# build: -# executor: java8-executor -# parallelism: 1 # This job doesn't benefit from parallelism -# steps: -# - log_environment -# - clone_cassandra -# - build_cassandra -# - run_eclipse_warnings -# - persist_to_workspace: -# root: /home/cassandra -# paths: -# - cassandra -# - .m2 -# -# j8_unit_tests: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests -# -# j8_jvm_dtests: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: test-jvm-dtest-forking -# -# utests_long: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: long-test -# -# utests_compression: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests: -# target: testclasslist-compression -# -# utests_stress: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: stress-test -# -# j8_dtests-with-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_with_vnodes -# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_with_vnodes -# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' -# -# j8_dtests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_without_vnodes -# run_dtests_extra_args: '--skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_without_vnodes -# pytest_extra_args: '--skip-resource-intensive-tests' -# -# j8_upgradetests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_upgradetests_without_vnodes -# run_dtests_extra_args: '--execute-upgrade-tests' -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# tests_filter_pattern: '^upgrade_tests' -# - run_dtests: -# file_tag: j8_upgradetests_without_vnodes -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# pytest_extra_args: '--execute-upgrade-tests' -# -# commands: -# log_environment: -# steps: -# - run: -# name: Log Environment Information -# command: | -# echo '*** id ***' -# id -# echo '*** cat /proc/cpuinfo ***' -# cat /proc/cpuinfo -# echo '*** free -m ***' -# free -m -# echo '*** df -m ***' -# df -m -# echo '*** ifconfig -a ***' -# ifconfig -a -# echo '*** uname -a ***' -# uname -a -# echo '*** mount ***' -# mount -# echo '*** env ***' -# env -# echo '*** java ***' -# which java -# java -version -# -# clone_cassandra: -# steps: -# - run: -# name: Clone Cassandra Repository (via git) -# command: | -# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra -# -# clone_dtest: -# steps: -# - run: -# name: Clone Cassandra dtest Repository (via git) -# command: | -# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest -# -# build_cassandra: -# steps: -# - run: -# name: Build Cassandra -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. -# for x in $(seq 1 3); do -# ${ANT_HOME}/bin/ant clean jar -# RETURN=\"$?\" -# if [ \"${RETURN}\" -eq \"0\" ]; then -# break -# fi -# done -# # Exit, if we didn't build successfully -# if [ \"${RETURN}\" -ne \"0\" ]; then -# echo \"Build failed with exit code: ${RETURN}\" -# exit ${RETURN} -# fi -# no_output_timeout: 15m -# -# run_eclipse_warnings: -# steps: -# - run: -# name: Run eclipse-warnings -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# ant eclipse-warnings -# -# create_junit_containers: -# steps: -# - run: -# name: Determine Unit Tests to Run -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# rm -fr ~/cassandra-dtest/upgrade_tests -# echo \"***java tests***\" -# -# # get all of our unit test filenames -# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt -# -# # split up the unit tests into groups based on the number of containers we have -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt -# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\" -# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# -# no_output_timeout: 15m -# -# run_junit_tests: -# parameters: -# target: -# type: string -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant clean <> -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# run_parallel_junit_tests: -# parameters: -# target: -# type: string -# default: testclasslist -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# create_venv: -# steps: -# - run: -# name: Configure virtualenv and python Dependencies -# command: | -# # note, this should be super quick as all dependencies should be pre-installed in the docker image -# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated -# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and -# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt -# pip3 freeze -# -# create_dtest_containers: -# parameters: -# file_tag: -# type: string -# run_dtests_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# tests_filter_pattern: -# type: string -# default: '' -# steps: -# - run: -# name: Determine Tests to Run (<>) -# no_output_timeout: 5m -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# cd cassandra-dtest -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# -# if [ -n '<>' ]; then -# export <> -# fi -# -# echo \"***Collected DTests (<>)***\" -# set -eo pipefail && ./run_dtests.py <> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<>_raw --cassandra-dir=../cassandra -# if [ -z '<>' ]; then -# mv /tmp/all_dtest_tests_<>_raw /tmp/all_dtest_tests_<> -# else -# grep -e '<>' /tmp/all_dtest_tests_<>_raw > /tmp/all_dtest_tests_<> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; } -# fi -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<> > /tmp/split_dtest_tests_<>.txt -# cat /tmp/split_dtest_tests_<>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<>_final.txt -# cat /tmp/split_dtest_tests_<>_final.txt -# -# run_dtests: -# parameters: -# file_tag: -# type: string -# pytest_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# steps: -# - run: -# name: Run dtests (<>) -# no_output_timeout: 15m -# command: | -# echo \"cat /tmp/split_dtest_tests_<>_final.txt\" -# cat /tmp/split_dtest_tests_<>_final.txt -# -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# if [ -n '<>' ]; then -# export <> -# fi -# -# java -version -# cd ~/cassandra-dtest -# mkdir -p /tmp/dtest -# -# echo \"env: $(env)\" -# echo \"** done env\" -# mkdir -p /tmp/results/dtests -# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee -# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<>_final.txt` -# set -o pipefail && cd ~/cassandra-dtest && pytest <> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt -# - store_test_results: -# path: /tmp/results -# - store_artifacts: -# path: /tmp/dtest -# destination: dtest_<> -# - store_artifacts: -# path: ~/cassandra-dtest/logs -# destination: dtest_<>_logs \ No newline at end of file diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES index 7016d9bd9b..e0c65f41ae 100644 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@ -1,5 +1,51 @@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean test-jvm-upgrade-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -109,37 +155,11 @@ jobs: - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\ - \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -280,6 +300,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean stress-test no_output_timeout: 15m - store_test_results: @@ -419,36 +442,11 @@ jobs: - run: name: Determine Tests to Run (j8_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: name: Run dtests (j8_with_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\ - \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -490,6 +488,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean test-jvm-dtest-forking no_output_timeout: 15m - store_test_results: @@ -533,6 +534,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean long-test no_output_timeout: 15m - store_test_results: @@ -644,6 +648,75 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + dtest_jars_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 build_and_run_tests: @@ -676,6 +749,16 @@ workflows: - utests_stress: requires: - start_utests_stress + - start_jvm_upgrade_dtest: + type: approval + requires: + - build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build - start_j8_dtests: type: approval requires: @@ -693,468 +776,3 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests - -# Original config.yml file: -# version: 2.1 -# -# default_env_vars: &default_env_vars -# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# ANT_HOME: /usr/share/ant -# LANG: en_US.UTF-8 -# KEEP_TEST_DIR: true -# DEFAULT_DIR: /home/cassandra/cassandra-dtest -# PYTHONIOENCODING: utf-8 -# PYTHONUNBUFFERED: true -# CASS_DRIVER_NO_EXTENSIONS: true -# CASS_DRIVER_NO_CYTHON: true -# #Skip all syncing to disk to avoid performance issues in flaky CI environments -# CASSANDRA_SKIP_SYNC: true -# DTEST_REPO: git://github.com/apache/cassandra-dtest.git -# DTEST_BRANCH: master -# CCM_MAX_HEAP_SIZE: 1024M -# CCM_HEAP_NEWSIZE: 256M -# -# j8_par_executor: &j8_par_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 4 -# -# j8_seq_executor: &j8_seq_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 1 # sequential, single container tests: no parallelism benefits -# -# with_dtests_jobs: &with_dtest_jobs -# jobs: -# - build -# # Java 8 unit tests will be run automatically -# - j8_unit_tests: -# requires: -# - build -# - j8_jvm_dtests: -# requires: -# - build -# # specialized unit tests (all run on request using Java 8) -# - start_utests_long: -# type: approval -# requires: -# - build -# - utests_long: -# requires: -# - start_utests_long -# - start_utests_compression: -# type: approval -# requires: -# - build -# - utests_compression: -# requires: -# - start_utests_compression -# - start_utests_stress: -# type: approval -# requires: -# - build -# - utests_stress: -# requires: -# - start_utests_stress -# # Java 8 dtests (on request) -# - start_j8_dtests: -# type: approval -# requires: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - start_j8_dtests -# - j8_dtests-no-vnodes: -# requires: -# - start_j8_dtests -# # Java 8 upgrade tests -# - start_upgrade_tests: -# type: approval -# requires: -# - build -# - j8_upgradetests-no-vnodes: -# requires: -# - start_upgrade_tests -# -# with_dtest_jobs_only: &with_dtest_jobs_only -# jobs: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - build -# - j8_dtests-no-vnodes: -# requires: -# - build -# -# workflows: -# version: 2 -# build_and_run_tests: *with_dtest_jobs -# #build_and_run_tests: *with_dtest_jobs_only -# -# executors: -# java8-executor: -# parameters: -# exec_resource_class: -# type: string -# default: medium -# docker: -# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 -# resource_class: << parameters.exec_resource_class >> -# working_directory: ~/ -# shell: /bin/bash -eo pipefail -l -# environment: -# <<: *default_env_vars -# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# -# jobs: -# build: -# executor: java8-executor -# parallelism: 1 # This job doesn't benefit from parallelism -# steps: -# - log_environment -# - clone_cassandra -# - build_cassandra -# - run_eclipse_warnings -# - persist_to_workspace: -# root: /home/cassandra -# paths: -# - cassandra -# - .m2 -# -# j8_unit_tests: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests -# -# j8_jvm_dtests: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: test-jvm-dtest-forking -# -# utests_long: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: long-test -# -# utests_compression: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests: -# target: testclasslist-compression -# -# utests_stress: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: stress-test -# -# j8_dtests-with-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_with_vnodes -# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_with_vnodes -# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' -# -# j8_dtests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_without_vnodes -# run_dtests_extra_args: '--skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_without_vnodes -# pytest_extra_args: '--skip-resource-intensive-tests' -# -# j8_upgradetests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_upgradetests_without_vnodes -# run_dtests_extra_args: '--execute-upgrade-tests' -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# tests_filter_pattern: '^upgrade_tests' -# - run_dtests: -# file_tag: j8_upgradetests_without_vnodes -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# pytest_extra_args: '--execute-upgrade-tests' -# -# commands: -# log_environment: -# steps: -# - run: -# name: Log Environment Information -# command: | -# echo '*** id ***' -# id -# echo '*** cat /proc/cpuinfo ***' -# cat /proc/cpuinfo -# echo '*** free -m ***' -# free -m -# echo '*** df -m ***' -# df -m -# echo '*** ifconfig -a ***' -# ifconfig -a -# echo '*** uname -a ***' -# uname -a -# echo '*** mount ***' -# mount -# echo '*** env ***' -# env -# echo '*** java ***' -# which java -# java -version -# -# clone_cassandra: -# steps: -# - run: -# name: Clone Cassandra Repository (via git) -# command: | -# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra -# -# clone_dtest: -# steps: -# - run: -# name: Clone Cassandra dtest Repository (via git) -# command: | -# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest -# -# build_cassandra: -# steps: -# - run: -# name: Build Cassandra -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. -# for x in $(seq 1 3); do -# ${ANT_HOME}/bin/ant clean jar -# RETURN=\"$?\" -# if [ \"${RETURN}\" -eq \"0\" ]; then -# break -# fi -# done -# # Exit, if we didn't build successfully -# if [ \"${RETURN}\" -ne \"0\" ]; then -# echo \"Build failed with exit code: ${RETURN}\" -# exit ${RETURN} -# fi -# no_output_timeout: 15m -# -# run_eclipse_warnings: -# steps: -# - run: -# name: Run eclipse-warnings -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# ant eclipse-warnings -# -# create_junit_containers: -# steps: -# - run: -# name: Determine Unit Tests to Run -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# rm -fr ~/cassandra-dtest/upgrade_tests -# echo \"***java tests***\" -# -# # get all of our unit test filenames -# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt -# -# # split up the unit tests into groups based on the number of containers we have -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt -# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\" -# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# -# no_output_timeout: 15m -# -# run_junit_tests: -# parameters: -# target: -# type: string -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant clean <> -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# run_parallel_junit_tests: -# parameters: -# target: -# type: string -# default: testclasslist -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# create_venv: -# steps: -# - run: -# name: Configure virtualenv and python Dependencies -# command: | -# # note, this should be super quick as all dependencies should be pre-installed in the docker image -# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated -# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and -# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt -# pip3 freeze -# -# create_dtest_containers: -# parameters: -# file_tag: -# type: string -# run_dtests_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# tests_filter_pattern: -# type: string -# default: '' -# steps: -# - run: -# name: Determine Tests to Run (<>) -# no_output_timeout: 5m -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# cd cassandra-dtest -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# -# if [ -n '<>' ]; then -# export <> -# fi -# -# echo \"***Collected DTests (<>)***\" -# set -eo pipefail && ./run_dtests.py <> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<>_raw --cassandra-dir=../cassandra -# if [ -z '<>' ]; then -# mv /tmp/all_dtest_tests_<>_raw /tmp/all_dtest_tests_<> -# else -# grep -e '<>' /tmp/all_dtest_tests_<>_raw > /tmp/all_dtest_tests_<> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; } -# fi -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<> > /tmp/split_dtest_tests_<>.txt -# cat /tmp/split_dtest_tests_<>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<>_final.txt -# cat /tmp/split_dtest_tests_<>_final.txt -# -# run_dtests: -# parameters: -# file_tag: -# type: string -# pytest_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# steps: -# - run: -# name: Run dtests (<>) -# no_output_timeout: 15m -# command: | -# echo \"cat /tmp/split_dtest_tests_<>_final.txt\" -# cat /tmp/split_dtest_tests_<>_final.txt -# -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# if [ -n '<>' ]; then -# export <> -# fi -# -# java -version -# cd ~/cassandra-dtest -# mkdir -p /tmp/dtest -# -# echo \"env: $(env)\" -# echo \"** done env\" -# mkdir -p /tmp/results/dtests -# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee -# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<>_final.txt` -# set -o pipefail && cd ~/cassandra-dtest && pytest <> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt -# - store_test_results: -# path: /tmp/results -# - store_artifacts: -# path: /tmp/dtest -# destination: dtest_<> -# - store_artifacts: -# path: ~/cassandra-dtest/logs -# destination: dtest_<>_logs \ No newline at end of file diff --git a/CHANGES.txt b/CHANGES.txt index 4a6867cc47..54f09a12e7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,7 @@ * Fix LegacyLayout to have same behavior as 2.x when handling unknown column names (CASSANDRA-15081) * Update nodetool help stop output (CASSANDRA-15401) Merged from 3.0: + * Run in-jvm upgrade dtests in circleci (CASSANDRA-15506) * Include updates to static column in mutation size calculations (CASSANDRA-15293) * Fix point-in-time recoevery ignoring timestamp of updates to static columns (CASSANDRA-15292) * GC logs are also put under $CASSANDRA_LOG_DIR (CASSANDRA-14306) diff --git a/build.xml b/build.xml index e81818e20f..eaedc7a207 100644 --- a/build.xml +++ b/build.xml @@ -67,6 +67,8 @@ + + @@ -1862,6 +1864,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java b/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java index a7ad400566..e5669109e1 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java +++ b/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java @@ -39,14 +39,18 @@ public class TestLocator public static void main(String[] args) throws Throwable { String outputFileName = defaultOutputFileName; - if (args.length == 1) + if (args.length >= 1) { outputFileName = args[0]; } + String testPackage = TestLocator.testPackage; + if (args.length == 2) + testPackage = args[1]; try (FileWriter fileWriter = new FileWriter(outputFileName); PrintWriter printWriter = new PrintWriter(fileWriter)) { printWriter.println("#!/bin/bash"); + printWriter.println("ret=0"); for (Class testClass : locateClasses(testPackage)) { for (Method method : testClass.getMethods()) @@ -57,8 +61,10 @@ public class TestLocator printWriter.println(String.format(testCommandFormat, testClass.getName(), method.getName())); + printWriter.println("if [ $? -ne 0 ]; then ret=1; fi"); } } + printWriter.println("exit $ret"); } }