diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml index f38bf28b57..3a1eff53a1 100644 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@ -23,6 +23,12 @@ j8_par_executor: &j8_par_executor #exec_resource_class: xlarge parallelism: 4 +j8_small_par_executor: &j8_small_par_executor + executor: + name: java8-executor + #exec_resource_class: xlarge + parallelism: 1 + j8_seq_executor: &j8_seq_executor executor: name: java8-executor @@ -129,12 +135,16 @@ jobs: - run_parallel_junit_tests j8_jvm_dtests: - <<: *j8_seq_executor + <<: *j8_small_par_executor steps: - attach_workspace: at: /home/cassandra - - run_junit_tests: - target: test-jvm-dtest-forking + - create_junit_containers: + classlistprefix: distributed + extra_filters: "| grep -v upgrade" + - log_environment + - run_parallel_junit_tests: + classlistprefix: distributed utests_long: <<: *j8_seq_executor @@ -271,9 +281,16 @@ commands: ant eclipse-warnings create_junit_containers: + parameters: + classlistprefix: + type: string + default: unit + extra_filters: + type: string + default: "" steps: - run: - name: Determine Unit Tests to Run + name: Determine <> Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -283,11 +300,11 @@ commands: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/<>/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/<>/;;g" | grep "Test\.java$" <> > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt @@ -328,6 +345,9 @@ commands: no_output_timeout: type: string default: 15m + classlistprefix: + type: string + default: unit steps: - run: name: Run Unit Tests (<>) @@ -337,7 +357,7 @@ commands: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra - ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<> no_output_timeout: <> - store_test_results: path: /tmp/cassandra/build/test/output/ diff --git a/.circleci/config-2_1.yml.high_res.patch b/.circleci/config-2_1.yml.high_res.patch index 22f434dd05..847a08f67b 100644 --- a/.circleci/config-2_1.yml.high_res.patch +++ b/.circleci/config-2_1.yml.high_res.patch @@ -10,7 +10,13 @@ --- > exec_resource_class: xlarge > parallelism: 100 -29c29 +29,30c29,30 +< #exec_resource_class: xlarge +< parallelism: 1 +--- +> exec_resource_class: xlarge +> parallelism: 2 +35c35 < #exec_resource_class: xlarge --- > exec_resource_class: xlarge diff --git a/.circleci/config.yml b/.circleci/config.yml index b1ed0ef494..ae582afed9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,7 +86,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -100,7 +100,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -170,12 +170,53 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (test-jvm-dtest-forking) + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra - ant clean test-jvm-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -256,7 +297,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -270,7 +311,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -357,36 +398,11 @@ jobs: - run: name: Determine Tests to Run (j8_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: name: Run dtests (j8_with_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\ - \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -440,37 +456,11 @@ jobs: - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\ - \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -637,452 +627,3 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests - -# Original config.yml file: -# version: 2.1 -# -# default_env_vars: &default_env_vars -# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# ANT_HOME: /usr/share/ant -# LANG: en_US.UTF-8 -# KEEP_TEST_DIR: true -# DEFAULT_DIR: /home/cassandra/cassandra-dtest -# PYTHONIOENCODING: utf-8 -# PYTHONUNBUFFERED: true -# CASS_DRIVER_NO_EXTENSIONS: true -# CASS_DRIVER_NO_CYTHON: true -# #Skip all syncing to disk to avoid performance issues in flaky CI environments -# CASSANDRA_SKIP_SYNC: true -# DTEST_REPO: git://github.com/apache/cassandra-dtest.git -# DTEST_BRANCH: master -# CCM_MAX_HEAP_SIZE: 1024M -# CCM_HEAP_NEWSIZE: 256M -# -# j8_par_executor: &j8_par_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 4 -# -# j8_seq_executor: &j8_seq_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 1 # sequential, single container tests: no parallelism benefits -# -# with_dtests_jobs: &with_dtest_jobs -# jobs: -# - build -# # Java 8 unit tests will be run automatically -# - j8_unit_tests: -# requires: -# - build -# - j8_jvm_dtests: -# requires: -# - build -# # specialized unit tests (all run on request using Java 8) -# - start_utests_long: -# type: approval -# requires: -# - build -# - utests_long: -# requires: -# - start_utests_long -# - start_utests_compression: -# type: approval -# requires: -# - build -# - utests_compression: -# requires: -# - start_utests_compression -# # Java 8 dtests (on request) -# - start_j8_dtests: -# type: approval -# requires: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - start_j8_dtests -# - j8_dtests-no-vnodes: -# requires: -# - start_j8_dtests -# # Java 8 upgrade tests -# - start_upgrade_tests: -# type: approval -# requires: -# - build -# - j8_upgradetests-no-vnodes: -# requires: -# - start_upgrade_tests -# -# with_dtest_jobs_only: &with_dtest_jobs_only -# jobs: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - build -# - j8_dtests-no-vnodes: -# requires: -# - build -# -# workflows: -# version: 2 -# build_and_run_tests: *with_dtest_jobs -# #build_and_run_tests: *with_dtest_jobs_only -# -# executors: -# java8-executor: -# parameters: -# exec_resource_class: -# type: string -# default: medium -# docker: -# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 -# resource_class: << parameters.exec_resource_class >> -# working_directory: ~/ -# shell: /bin/bash -eo pipefail -l -# environment: -# <<: *default_env_vars -# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# -# jobs: -# build: -# executor: java8-executor -# parallelism: 1 # This job doesn't benefit from parallelism -# steps: -# - log_environment -# - clone_cassandra -# - build_cassandra -# - persist_to_workspace: -# root: /home/cassandra -# paths: -# - cassandra -# - .m2 -# -# j8_unit_tests: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests -# -# j8_jvm_dtests: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: test-jvm-dtest-forking -# -# utests_long: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: long-test -# -# utests_compression: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests: -# target: testclasslist-compression -# -# j8_dtests-with-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_with_vnodes -# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_with_vnodes -# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' -# -# j8_dtests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_without_vnodes -# run_dtests_extra_args: '--skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_without_vnodes -# pytest_extra_args: '--skip-resource-intensive-tests' -# -# j8_upgradetests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_upgradetests_without_vnodes -# run_dtests_extra_args: '--execute-upgrade-tests' -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# tests_filter_pattern: '^upgrade_tests' -# - run_dtests: -# file_tag: j8_upgradetests_without_vnodes -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# pytest_extra_args: '--execute-upgrade-tests' -# -# commands: -# log_environment: -# steps: -# - run: -# name: Log Environment Information -# command: | -# echo '*** id ***' -# id -# echo '*** cat /proc/cpuinfo ***' -# cat /proc/cpuinfo -# echo '*** free -m ***' -# free -m -# echo '*** df -m ***' -# df -m -# echo '*** ifconfig -a ***' -# ifconfig -a -# echo '*** uname -a ***' -# uname -a -# echo '*** mount ***' -# mount -# echo '*** env ***' -# env -# echo '*** java ***' -# which java -# java -version -# -# clone_cassandra: -# steps: -# - run: -# name: Clone Cassandra Repository (via git) -# command: | -# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra -# -# clone_dtest: -# steps: -# - run: -# name: Clone Cassandra dtest Repository (via git) -# command: | -# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest -# -# build_cassandra: -# steps: -# - run: -# name: Build Cassandra -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. -# for x in $(seq 1 3); do -# ${ANT_HOME}/bin/ant clean jar -# RETURN=\"$?\" -# if [ \"${RETURN}\" -eq \"0\" ]; then -# break -# fi -# done -# # Exit, if we didn't build successfully -# if [ \"${RETURN}\" -ne \"0\" ]; then -# echo \"Build failed with exit code: ${RETURN}\" -# exit ${RETURN} -# fi -# no_output_timeout: 15m -# -# run_eclipse_warnings: -# steps: -# - run: -# name: Run eclipse-warnings -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# ant eclipse-warnings -# -# create_junit_containers: -# steps: -# - run: -# name: Determine Unit Tests to Run -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# rm -fr ~/cassandra-dtest/upgrade_tests -# echo \"***java tests***\" -# -# # get all of our unit test filenames -# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt -# -# # split up the unit tests into groups based on the number of containers we have -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt -# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\" -# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# -# no_output_timeout: 15m -# -# run_junit_tests: -# parameters: -# target: -# type: string -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant clean <> -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# run_parallel_junit_tests: -# parameters: -# target: -# type: string -# default: testclasslist -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# create_venv: -# steps: -# - run: -# name: Configure virtualenv and python Dependencies -# command: | -# # note, this should be super quick as all dependencies should be pre-installed in the docker image -# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated -# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and -# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt -# pip3 freeze -# -# create_dtest_containers: -# parameters: -# file_tag: -# type: string -# run_dtests_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# tests_filter_pattern: -# type: string -# default: '' -# steps: -# - run: -# name: Determine Tests to Run (<>) -# no_output_timeout: 5m -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# cd cassandra-dtest -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# -# if [ -n '<>' ]; then -# export <> -# fi -# -# echo \"***Collected DTests (<>)***\" -# set -eo pipefail && ./run_dtests.py <> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<>_raw --cassandra-dir=../cassandra -# if [ -z '<>' ]; then -# mv /tmp/all_dtest_tests_<>_raw /tmp/all_dtest_tests_<> -# else -# grep -e '<>' /tmp/all_dtest_tests_<>_raw > /tmp/all_dtest_tests_<> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; } -# fi -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<> > /tmp/split_dtest_tests_<>.txt -# cat /tmp/split_dtest_tests_<>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<>_final.txt -# cat /tmp/split_dtest_tests_<>_final.txt -# -# run_dtests: -# parameters: -# file_tag: -# type: string -# pytest_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# steps: -# - run: -# name: Run dtests (<>) -# no_output_timeout: 15m -# command: | -# echo \"cat /tmp/split_dtest_tests_<>_final.txt\" -# cat /tmp/split_dtest_tests_<>_final.txt -# -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# if [ -n '<>' ]; then -# export <> -# fi -# -# java -version -# cd ~/cassandra-dtest -# mkdir -p /tmp/dtest -# -# echo \"env: $(env)\" -# echo \"** done env\" -# mkdir -p /tmp/results/dtests -# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee -# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<>_final.txt` -# set -o pipefail && cd ~/cassandra-dtest && pytest <> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt -# - store_test_results: -# path: /tmp/results -# - store_artifacts: -# path: /tmp/dtest -# destination: dtest_<> -# - store_artifacts: -# path: ~/cassandra-dtest/logs -# destination: dtest_<>_logs diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES index 4bbef27d00..e5ed2749a2 100644 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@ -86,7 +86,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -100,7 +100,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -165,17 +165,58 @@ jobs: resource_class: xlarge working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 1 + parallelism: 2 steps: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (test-jvm-dtest-forking) + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra - ant clean test-jvm-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -256,7 +297,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -270,7 +311,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -357,36 +398,11 @@ jobs: - run: name: Determine Tests to Run (j8_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: name: Run dtests (j8_with_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\ - \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -440,37 +456,11 @@ jobs: - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\ - \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -637,452 +627,3 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests - -# Original config.yml file: -# version: 2.1 -# -# default_env_vars: &default_env_vars -# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# ANT_HOME: /usr/share/ant -# LANG: en_US.UTF-8 -# KEEP_TEST_DIR: true -# DEFAULT_DIR: /home/cassandra/cassandra-dtest -# PYTHONIOENCODING: utf-8 -# PYTHONUNBUFFERED: true -# CASS_DRIVER_NO_EXTENSIONS: true -# CASS_DRIVER_NO_CYTHON: true -# #Skip all syncing to disk to avoid performance issues in flaky CI environments -# CASSANDRA_SKIP_SYNC: true -# DTEST_REPO: git://github.com/apache/cassandra-dtest.git -# DTEST_BRANCH: master -# CCM_MAX_HEAP_SIZE: 2048M -# CCM_HEAP_NEWSIZE: 512M -# -# j8_par_executor: &j8_par_executor -# executor: -# name: java8-executor -# exec_resource_class: xlarge -# parallelism: 100 -# -# j8_seq_executor: &j8_seq_executor -# executor: -# name: java8-executor -# exec_resource_class: xlarge -# parallelism: 1 # sequential, single container tests: no parallelism benefits -# -# with_dtests_jobs: &with_dtest_jobs -# jobs: -# - build -# # Java 8 unit tests will be run automatically -# - j8_unit_tests: -# requires: -# - build -# - j8_jvm_dtests: -# requires: -# - build -# # specialized unit tests (all run on request using Java 8) -# - start_utests_long: -# type: approval -# requires: -# - build -# - utests_long: -# requires: -# - start_utests_long -# - start_utests_compression: -# type: approval -# requires: -# - build -# - utests_compression: -# requires: -# - start_utests_compression -# # Java 8 dtests (on request) -# - start_j8_dtests: -# type: approval -# requires: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - start_j8_dtests -# - j8_dtests-no-vnodes: -# requires: -# - start_j8_dtests -# # Java 8 upgrade tests -# - start_upgrade_tests: -# type: approval -# requires: -# - build -# - j8_upgradetests-no-vnodes: -# requires: -# - start_upgrade_tests -# -# with_dtest_jobs_only: &with_dtest_jobs_only -# jobs: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - build -# - j8_dtests-no-vnodes: -# requires: -# - build -# -# workflows: -# version: 2 -# build_and_run_tests: *with_dtest_jobs -# #build_and_run_tests: *with_dtest_jobs_only -# -# executors: -# java8-executor: -# parameters: -# exec_resource_class: -# type: string -# default: medium -# docker: -# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 -# resource_class: << parameters.exec_resource_class >> -# working_directory: ~/ -# shell: /bin/bash -eo pipefail -l -# environment: -# <<: *default_env_vars -# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# -# jobs: -# build: -# executor: java8-executor -# parallelism: 1 # This job doesn't benefit from parallelism -# steps: -# - log_environment -# - clone_cassandra -# - build_cassandra -# - persist_to_workspace: -# root: /home/cassandra -# paths: -# - cassandra -# - .m2 -# -# j8_unit_tests: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests -# -# j8_jvm_dtests: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: test-jvm-dtest-forking -# -# utests_long: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: long-test -# -# utests_compression: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests: -# target: testclasslist-compression -# -# j8_dtests-with-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_with_vnodes -# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_with_vnodes -# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' -# -# j8_dtests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_without_vnodes -# run_dtests_extra_args: '--skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_without_vnodes -# pytest_extra_args: '--skip-resource-intensive-tests' -# -# j8_upgradetests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_upgradetests_without_vnodes -# run_dtests_extra_args: '--execute-upgrade-tests' -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# tests_filter_pattern: '^upgrade_tests' -# - run_dtests: -# file_tag: j8_upgradetests_without_vnodes -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# pytest_extra_args: '--execute-upgrade-tests' -# -# commands: -# log_environment: -# steps: -# - run: -# name: Log Environment Information -# command: | -# echo '*** id ***' -# id -# echo '*** cat /proc/cpuinfo ***' -# cat /proc/cpuinfo -# echo '*** free -m ***' -# free -m -# echo '*** df -m ***' -# df -m -# echo '*** ifconfig -a ***' -# ifconfig -a -# echo '*** uname -a ***' -# uname -a -# echo '*** mount ***' -# mount -# echo '*** env ***' -# env -# echo '*** java ***' -# which java -# java -version -# -# clone_cassandra: -# steps: -# - run: -# name: Clone Cassandra Repository (via git) -# command: | -# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra -# -# clone_dtest: -# steps: -# - run: -# name: Clone Cassandra dtest Repository (via git) -# command: | -# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest -# -# build_cassandra: -# steps: -# - run: -# name: Build Cassandra -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. -# for x in $(seq 1 3); do -# ${ANT_HOME}/bin/ant clean jar -# RETURN=\"$?\" -# if [ \"${RETURN}\" -eq \"0\" ]; then -# break -# fi -# done -# # Exit, if we didn't build successfully -# if [ \"${RETURN}\" -ne \"0\" ]; then -# echo \"Build failed with exit code: ${RETURN}\" -# exit ${RETURN} -# fi -# no_output_timeout: 15m -# -# run_eclipse_warnings: -# steps: -# - run: -# name: Run eclipse-warnings -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# ant eclipse-warnings -# -# create_junit_containers: -# steps: -# - run: -# name: Determine Unit Tests to Run -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# rm -fr ~/cassandra-dtest/upgrade_tests -# echo \"***java tests***\" -# -# # get all of our unit test filenames -# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt -# -# # split up the unit tests into groups based on the number of containers we have -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt -# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\" -# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# -# no_output_timeout: 15m -# -# run_junit_tests: -# parameters: -# target: -# type: string -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant clean <> -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# run_parallel_junit_tests: -# parameters: -# target: -# type: string -# default: testclasslist -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# create_venv: -# steps: -# - run: -# name: Configure virtualenv and python Dependencies -# command: | -# # note, this should be super quick as all dependencies should be pre-installed in the docker image -# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated -# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and -# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt -# pip3 freeze -# -# create_dtest_containers: -# parameters: -# file_tag: -# type: string -# run_dtests_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# tests_filter_pattern: -# type: string -# default: '' -# steps: -# - run: -# name: Determine Tests to Run (<>) -# no_output_timeout: 5m -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# cd cassandra-dtest -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# -# if [ -n '<>' ]; then -# export <> -# fi -# -# echo \"***Collected DTests (<>)***\" -# set -eo pipefail && ./run_dtests.py <> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<>_raw --cassandra-dir=../cassandra -# if [ -z '<>' ]; then -# mv /tmp/all_dtest_tests_<>_raw /tmp/all_dtest_tests_<> -# else -# grep -e '<>' /tmp/all_dtest_tests_<>_raw > /tmp/all_dtest_tests_<> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; } -# fi -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<> > /tmp/split_dtest_tests_<>.txt -# cat /tmp/split_dtest_tests_<>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<>_final.txt -# cat /tmp/split_dtest_tests_<>_final.txt -# -# run_dtests: -# parameters: -# file_tag: -# type: string -# pytest_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# steps: -# - run: -# name: Run dtests (<>) -# no_output_timeout: 15m -# command: | -# echo \"cat /tmp/split_dtest_tests_<>_final.txt\" -# cat /tmp/split_dtest_tests_<>_final.txt -# -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# if [ -n '<>' ]; then -# export <> -# fi -# -# java -version -# cd ~/cassandra-dtest -# mkdir -p /tmp/dtest -# -# echo \"env: $(env)\" -# echo \"** done env\" -# mkdir -p /tmp/results/dtests -# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee -# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<>_final.txt` -# set -o pipefail && cd ~/cassandra-dtest && pytest <> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt -# - store_test_results: -# path: /tmp/results -# - store_artifacts: -# path: /tmp/dtest -# destination: dtest_<> -# - store_artifacts: -# path: ~/cassandra-dtest/logs -# destination: dtest_<>_logs diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES index b1ed0ef494..ae582afed9 100644 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@ -86,7 +86,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -100,7 +100,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -170,12 +170,53 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (test-jvm-dtest-forking) + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra - ant clean test-jvm-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -256,7 +297,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -270,7 +311,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -357,36 +398,11 @@ jobs: - run: name: Determine Tests to Run (j8_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: name: Run dtests (j8_with_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\ - \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -440,37 +456,11 @@ jobs: - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\ - \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -637,452 +627,3 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests - -# Original config.yml file: -# version: 2.1 -# -# default_env_vars: &default_env_vars -# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# ANT_HOME: /usr/share/ant -# LANG: en_US.UTF-8 -# KEEP_TEST_DIR: true -# DEFAULT_DIR: /home/cassandra/cassandra-dtest -# PYTHONIOENCODING: utf-8 -# PYTHONUNBUFFERED: true -# CASS_DRIVER_NO_EXTENSIONS: true -# CASS_DRIVER_NO_CYTHON: true -# #Skip all syncing to disk to avoid performance issues in flaky CI environments -# CASSANDRA_SKIP_SYNC: true -# DTEST_REPO: git://github.com/apache/cassandra-dtest.git -# DTEST_BRANCH: master -# CCM_MAX_HEAP_SIZE: 1024M -# CCM_HEAP_NEWSIZE: 256M -# -# j8_par_executor: &j8_par_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 4 -# -# j8_seq_executor: &j8_seq_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 1 # sequential, single container tests: no parallelism benefits -# -# with_dtests_jobs: &with_dtest_jobs -# jobs: -# - build -# # Java 8 unit tests will be run automatically -# - j8_unit_tests: -# requires: -# - build -# - j8_jvm_dtests: -# requires: -# - build -# # specialized unit tests (all run on request using Java 8) -# - start_utests_long: -# type: approval -# requires: -# - build -# - utests_long: -# requires: -# - start_utests_long -# - start_utests_compression: -# type: approval -# requires: -# - build -# - utests_compression: -# requires: -# - start_utests_compression -# # Java 8 dtests (on request) -# - start_j8_dtests: -# type: approval -# requires: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - start_j8_dtests -# - j8_dtests-no-vnodes: -# requires: -# - start_j8_dtests -# # Java 8 upgrade tests -# - start_upgrade_tests: -# type: approval -# requires: -# - build -# - j8_upgradetests-no-vnodes: -# requires: -# - start_upgrade_tests -# -# with_dtest_jobs_only: &with_dtest_jobs_only -# jobs: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - build -# - j8_dtests-no-vnodes: -# requires: -# - build -# -# workflows: -# version: 2 -# build_and_run_tests: *with_dtest_jobs -# #build_and_run_tests: *with_dtest_jobs_only -# -# executors: -# java8-executor: -# parameters: -# exec_resource_class: -# type: string -# default: medium -# docker: -# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 -# resource_class: << parameters.exec_resource_class >> -# working_directory: ~/ -# shell: /bin/bash -eo pipefail -l -# environment: -# <<: *default_env_vars -# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# -# jobs: -# build: -# executor: java8-executor -# parallelism: 1 # This job doesn't benefit from parallelism -# steps: -# - log_environment -# - clone_cassandra -# - build_cassandra -# - persist_to_workspace: -# root: /home/cassandra -# paths: -# - cassandra -# - .m2 -# -# j8_unit_tests: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests -# -# j8_jvm_dtests: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: test-jvm-dtest-forking -# -# utests_long: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: long-test -# -# utests_compression: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests: -# target: testclasslist-compression -# -# j8_dtests-with-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_with_vnodes -# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_with_vnodes -# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' -# -# j8_dtests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_without_vnodes -# run_dtests_extra_args: '--skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_without_vnodes -# pytest_extra_args: '--skip-resource-intensive-tests' -# -# j8_upgradetests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_upgradetests_without_vnodes -# run_dtests_extra_args: '--execute-upgrade-tests' -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# tests_filter_pattern: '^upgrade_tests' -# - run_dtests: -# file_tag: j8_upgradetests_without_vnodes -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# pytest_extra_args: '--execute-upgrade-tests' -# -# commands: -# log_environment: -# steps: -# - run: -# name: Log Environment Information -# command: | -# echo '*** id ***' -# id -# echo '*** cat /proc/cpuinfo ***' -# cat /proc/cpuinfo -# echo '*** free -m ***' -# free -m -# echo '*** df -m ***' -# df -m -# echo '*** ifconfig -a ***' -# ifconfig -a -# echo '*** uname -a ***' -# uname -a -# echo '*** mount ***' -# mount -# echo '*** env ***' -# env -# echo '*** java ***' -# which java -# java -version -# -# clone_cassandra: -# steps: -# - run: -# name: Clone Cassandra Repository (via git) -# command: | -# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra -# -# clone_dtest: -# steps: -# - run: -# name: Clone Cassandra dtest Repository (via git) -# command: | -# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest -# -# build_cassandra: -# steps: -# - run: -# name: Build Cassandra -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. -# for x in $(seq 1 3); do -# ${ANT_HOME}/bin/ant clean jar -# RETURN=\"$?\" -# if [ \"${RETURN}\" -eq \"0\" ]; then -# break -# fi -# done -# # Exit, if we didn't build successfully -# if [ \"${RETURN}\" -ne \"0\" ]; then -# echo \"Build failed with exit code: ${RETURN}\" -# exit ${RETURN} -# fi -# no_output_timeout: 15m -# -# run_eclipse_warnings: -# steps: -# - run: -# name: Run eclipse-warnings -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# ant eclipse-warnings -# -# create_junit_containers: -# steps: -# - run: -# name: Determine Unit Tests to Run -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# rm -fr ~/cassandra-dtest/upgrade_tests -# echo \"***java tests***\" -# -# # get all of our unit test filenames -# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt -# -# # split up the unit tests into groups based on the number of containers we have -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt -# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\" -# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# -# no_output_timeout: 15m -# -# run_junit_tests: -# parameters: -# target: -# type: string -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant clean <> -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# run_parallel_junit_tests: -# parameters: -# target: -# type: string -# default: testclasslist -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# create_venv: -# steps: -# - run: -# name: Configure virtualenv and python Dependencies -# command: | -# # note, this should be super quick as all dependencies should be pre-installed in the docker image -# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated -# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and -# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt -# pip3 freeze -# -# create_dtest_containers: -# parameters: -# file_tag: -# type: string -# run_dtests_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# tests_filter_pattern: -# type: string -# default: '' -# steps: -# - run: -# name: Determine Tests to Run (<>) -# no_output_timeout: 5m -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# cd cassandra-dtest -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# -# if [ -n '<>' ]; then -# export <> -# fi -# -# echo \"***Collected DTests (<>)***\" -# set -eo pipefail && ./run_dtests.py <> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<>_raw --cassandra-dir=../cassandra -# if [ -z '<>' ]; then -# mv /tmp/all_dtest_tests_<>_raw /tmp/all_dtest_tests_<> -# else -# grep -e '<>' /tmp/all_dtest_tests_<>_raw > /tmp/all_dtest_tests_<> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; } -# fi -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<> > /tmp/split_dtest_tests_<>.txt -# cat /tmp/split_dtest_tests_<>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<>_final.txt -# cat /tmp/split_dtest_tests_<>_final.txt -# -# run_dtests: -# parameters: -# file_tag: -# type: string -# pytest_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# steps: -# - run: -# name: Run dtests (<>) -# no_output_timeout: 15m -# command: | -# echo \"cat /tmp/split_dtest_tests_<>_final.txt\" -# cat /tmp/split_dtest_tests_<>_final.txt -# -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# if [ -n '<>' ]; then -# export <> -# fi -# -# java -version -# cd ~/cassandra-dtest -# mkdir -p /tmp/dtest -# -# echo \"env: $(env)\" -# echo \"** done env\" -# mkdir -p /tmp/results/dtests -# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee -# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<>_final.txt` -# set -o pipefail && cd ~/cassandra-dtest && pytest <> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt -# - store_test_results: -# path: /tmp/results -# - store_artifacts: -# path: /tmp/dtest -# destination: dtest_<> -# - store_artifacts: -# path: ~/cassandra-dtest/logs -# destination: dtest_<>_logs