From 5f5721226788a9db73ad37a48bd02a66c84b7e5c Mon Sep 17 00:00:00 2001 From: Marcus Eriksson Date: Wed, 15 Jan 2020 10:40:03 +0100 Subject: [PATCH] Run in-jvm upgrade dtests in circle ci Patch by marcuse; reviewed by Alex Petrov for CASSANDRA-15506 --- .circleci/config-2_1.yml | 74 + .circleci/config.yml | 1260 ++++++----------- .circleci/config.yml.HIGHRES | 1260 ++++++----------- .circleci/config.yml.LOWRES | 1260 ++++++----------- CHANGES.txt | 1 + build.xml | 35 + .../distributed/test/TestLocator.java | 8 +- 7 files changed, 1452 insertions(+), 2446 deletions(-) diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml index 859100d194..a4410fe72a 100644 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@ -54,6 +54,16 @@ with_dtests_jobs: &with_dtest_jobs - utests_compression: requires: - start_utests_compression + - start_jvm_upgrade_dtest: + type: approval + requires: + - build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build # Java 8 dtests (on request) - start_j8_dtests: type: approval @@ -120,6 +130,18 @@ jobs: - cassandra - .m2 + dtest_jars_build: + executor: java8-executor + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - build_cassandra_dtest_jars + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + j8_unit_tests: <<: *j8_par_executor steps: @@ -137,6 +159,14 @@ jobs: - run_junit_tests: target: test-jvm-dtest-forking + j8_jvm_upgrade_dtests: + <<: *j8_seq_executor + steps: + - attach_workspace: + at: /home/cassandra + - run_junit_tests: + target: test-jvm-upgrade-dtest-forking + utests_long: <<: *j8_seq_executor steps: @@ -262,6 +292,47 @@ commands: fi no_output_timeout: 15m + build_cassandra_dtest_jars: + steps: + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + run_eclipse_warnings: steps: - run: @@ -310,6 +381,9 @@ commands: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant clean <> no_output_timeout: <> - store_test_results: diff --git a/.circleci/config.yml b/.circleci/config.yml index 0d56e23c84..d80b78d8de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,51 @@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean test-jvm-upgrade-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -81,343 +127,6 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_unit_tests: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine Unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_jvm_dtests: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Run Unit Tests (test-jvm-dtest-forking) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant clean test-jvm-dtest-forking - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_long: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Run Unit Tests (long-test) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant clean long-test - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_compression: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine Unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist-compression) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-with-vnodes: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - - run: - name: Run dtests (j8_with_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\ - \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_with_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_with_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests-no-vnodes: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -446,37 +155,11 @@ jobs: - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\ - \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -601,11 +284,408 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_unit_tests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine Unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean test-jvm-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_long: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (long-test) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean long-test + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_compression: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine Unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist-compression) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + dtest_jars_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 build_and_run_tests: jobs: - build + - start_jvm_upgrade_dtest: + type: approval + requires: + - build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build - j8_unit_tests: requires: - build @@ -643,453 +723,3 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests - -# Original config.yml file: -# version: 2.1 -# -# default_env_vars: &default_env_vars -# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# ANT_HOME: /usr/share/ant -# LANG: en_US.UTF-8 -# KEEP_TEST_DIR: true -# DEFAULT_DIR: /home/cassandra/cassandra-dtest -# PYTHONIOENCODING: utf-8 -# PYTHONUNBUFFERED: true -# CASS_DRIVER_NO_EXTENSIONS: true -# CASS_DRIVER_NO_CYTHON: true -# #Skip all syncing to disk to avoid performance issues in flaky CI environments -# CASSANDRA_SKIP_SYNC: true -# DTEST_REPO: git://github.com/apache/cassandra-dtest.git -# DTEST_BRANCH: master -# CCM_MAX_HEAP_SIZE: 1024M -# CCM_HEAP_NEWSIZE: 256M -# -# j8_par_executor: &j8_par_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 4 -# -# j8_seq_executor: &j8_seq_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 1 # sequential, single container tests: no parallelism benefits -# -# with_dtests_jobs: &with_dtest_jobs -# jobs: -# - build -# # Java 8 unit tests will be run automatically -# - j8_unit_tests: -# requires: -# - build -# - j8_jvm_dtests: -# requires: -# - build -# # specialized unit tests (all run on request using Java 8) -# - start_utests_long: -# type: approval -# requires: -# - build -# - utests_long: -# requires: -# - start_utests_long -# - start_utests_compression: -# type: approval -# requires: -# - build -# - utests_compression: -# requires: -# - start_utests_compression -# # Java 8 dtests (on request) -# - start_j8_dtests: -# type: approval -# requires: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - start_j8_dtests -# - j8_dtests-no-vnodes: -# requires: -# - start_j8_dtests -# # Java 8 upgrade tests -# - start_upgrade_tests: -# type: approval -# requires: -# - build -# - j8_upgradetests-no-vnodes: -# requires: -# - start_upgrade_tests -# -# with_dtest_jobs_only: &with_dtest_jobs_only -# jobs: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - build -# - j8_dtests-no-vnodes: -# requires: -# - build -# -# workflows: -# version: 2 -# build_and_run_tests: *with_dtest_jobs -# #build_and_run_tests: *with_dtest_jobs_only -# -# executors: -# java8-executor: -# parameters: -# exec_resource_class: -# type: string -# default: medium -# docker: -# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 -# resource_class: << parameters.exec_resource_class >> -# working_directory: ~/ -# shell: /bin/bash -eo pipefail -l -# environment: -# <<: *default_env_vars -# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# -# jobs: -# build: -# executor: java8-executor -# parallelism: 1 # This job doesn't benefit from parallelism -# steps: -# - log_environment -# - clone_cassandra -# - build_cassandra -# - run_eclipse_warnings -# - persist_to_workspace: -# root: /home/cassandra -# paths: -# - cassandra -# - .m2 -# -# j8_unit_tests: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests -# -# j8_jvm_dtests: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: test-jvm-dtest-forking -# -# utests_long: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: long-test -# -# utests_compression: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests: -# target: testclasslist-compression -# -# j8_dtests-with-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_with_vnodes -# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_with_vnodes -# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' -# -# j8_dtests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_without_vnodes -# run_dtests_extra_args: '--skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_without_vnodes -# pytest_extra_args: '--skip-resource-intensive-tests' -# -# j8_upgradetests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_upgradetests_without_vnodes -# run_dtests_extra_args: '--execute-upgrade-tests' -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# tests_filter_pattern: '^upgrade_tests' -# - run_dtests: -# file_tag: j8_upgradetests_without_vnodes -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# pytest_extra_args: '--execute-upgrade-tests' -# -# commands: -# log_environment: -# steps: -# - run: -# name: Log Environment Information -# command: | -# echo '*** id ***' -# id -# echo '*** cat /proc/cpuinfo ***' -# cat /proc/cpuinfo -# echo '*** free -m ***' -# free -m -# echo '*** df -m ***' -# df -m -# echo '*** ifconfig -a ***' -# ifconfig -a -# echo '*** uname -a ***' -# uname -a -# echo '*** mount ***' -# mount -# echo '*** env ***' -# env -# echo '*** java ***' -# which java -# java -version -# -# clone_cassandra: -# steps: -# - run: -# name: Clone Cassandra Repository (via git) -# command: | -# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra -# -# clone_dtest: -# steps: -# - run: -# name: Clone Cassandra dtest Repository (via git) -# command: | -# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest -# -# build_cassandra: -# steps: -# - run: -# name: Build Cassandra -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. -# for x in $(seq 1 3); do -# ${ANT_HOME}/bin/ant clean jar -# RETURN=\"$?\" -# if [ \"${RETURN}\" -eq \"0\" ]; then -# break -# fi -# done -# # Exit, if we didn't build successfully -# if [ \"${RETURN}\" -ne \"0\" ]; then -# echo \"Build failed with exit code: ${RETURN}\" -# exit ${RETURN} -# fi -# no_output_timeout: 15m -# -# run_eclipse_warnings: -# steps: -# - run: -# name: Run eclipse-warnings -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# ant eclipse-warnings -# -# create_junit_containers: -# steps: -# - run: -# name: Determine Unit Tests to Run -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# rm -fr ~/cassandra-dtest/upgrade_tests -# echo \"***java tests***\" -# -# # get all of our unit test filenames -# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt -# -# # split up the unit tests into groups based on the number of containers we have -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt -# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\" -# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# -# no_output_timeout: 15m -# -# run_junit_tests: -# parameters: -# target: -# type: string -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant clean <> -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# run_parallel_junit_tests: -# parameters: -# target: -# type: string -# default: testclasslist -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# create_venv: -# steps: -# - run: -# name: Configure virtualenv and python Dependencies -# command: | -# # note, this should be super quick as all dependencies should be pre-installed in the docker image -# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated -# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and -# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt -# pip3 freeze -# -# create_dtest_containers: -# parameters: -# file_tag: -# type: string -# run_dtests_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# tests_filter_pattern: -# type: string -# default: '' -# steps: -# - run: -# name: Determine Tests to Run (<>) -# no_output_timeout: 5m -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# cd cassandra-dtest -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# -# if [ -n '<>' ]; then -# export <> -# fi -# -# echo \"***Collected DTests (<>)***\" -# set -eo pipefail && ./run_dtests.py <> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<>_raw --cassandra-dir=../cassandra -# if [ -z '<>' ]; then -# mv /tmp/all_dtest_tests_<>_raw /tmp/all_dtest_tests_<> -# else -# grep -e '<>' /tmp/all_dtest_tests_<>_raw > /tmp/all_dtest_tests_<> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; } -# fi -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<> > /tmp/split_dtest_tests_<>.txt -# cat /tmp/split_dtest_tests_<>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<>_final.txt -# cat /tmp/split_dtest_tests_<>_final.txt -# -# run_dtests: -# parameters: -# file_tag: -# type: string -# pytest_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# steps: -# - run: -# name: Run dtests (<>) -# no_output_timeout: 15m -# command: | -# echo \"cat /tmp/split_dtest_tests_<>_final.txt\" -# cat /tmp/split_dtest_tests_<>_final.txt -# -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# if [ -n '<>' ]; then -# export <> -# fi -# -# java -version -# cd ~/cassandra-dtest -# mkdir -p /tmp/dtest -# -# echo \"env: $(env)\" -# echo \"** done env\" -# mkdir -p /tmp/results/dtests -# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee -# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<>_final.txt` -# set -o pipefail && cd ~/cassandra-dtest && pytest <> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt -# - store_test_results: -# path: /tmp/results -# - store_artifacts: -# path: /tmp/dtest -# destination: dtest_<> -# - store_artifacts: -# path: ~/cassandra-dtest/logs -# destination: dtest_<>_logs \ No newline at end of file diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES index ab38b4510d..94ca5b2df5 100644 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@ -1,5 +1,51 @@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean test-jvm-upgrade-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -81,343 +127,6 @@ jobs: - CCM_HEAP_NEWSIZE: 512M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_unit_tests: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine Unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_jvm_dtests: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Run Unit Tests (test-jvm-dtest-forking) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant clean test-jvm-dtest-forking - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_long: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Run Unit Tests (long-test) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant clean long-test - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_compression: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine Unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist-compression) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-with-vnodes: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: xlarge - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 100 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - - run: - name: Run dtests (j8_with_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\ - \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_with_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_with_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 2048M - - CCM_HEAP_NEWSIZE: 512M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests-no-vnodes: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -446,37 +155,11 @@ jobs: - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\ - \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -601,11 +284,408 @@ jobs: - CCM_HEAP_NEWSIZE: 512M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_unit_tests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine Unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean test-jvm-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_long: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (long-test) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean long-test + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_compression: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine Unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist-compression) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + dtest_jars_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 build_and_run_tests: jobs: - build + - start_jvm_upgrade_dtest: + type: approval + requires: + - build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build - j8_unit_tests: requires: - build @@ -643,453 +723,3 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests - -# Original config.yml file: -# version: 2.1 -# -# default_env_vars: &default_env_vars -# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# ANT_HOME: /usr/share/ant -# LANG: en_US.UTF-8 -# KEEP_TEST_DIR: true -# DEFAULT_DIR: /home/cassandra/cassandra-dtest -# PYTHONIOENCODING: utf-8 -# PYTHONUNBUFFERED: true -# CASS_DRIVER_NO_EXTENSIONS: true -# CASS_DRIVER_NO_CYTHON: true -# #Skip all syncing to disk to avoid performance issues in flaky CI environments -# CASSANDRA_SKIP_SYNC: true -# DTEST_REPO: git://github.com/apache/cassandra-dtest.git -# DTEST_BRANCH: master -# CCM_MAX_HEAP_SIZE: 2048M -# CCM_HEAP_NEWSIZE: 512M -# -# j8_par_executor: &j8_par_executor -# executor: -# name: java8-executor -# exec_resource_class: xlarge -# parallelism: 100 -# -# j8_seq_executor: &j8_seq_executor -# executor: -# name: java8-executor -# exec_resource_class: xlarge -# parallelism: 1 # sequential, single container tests: no parallelism benefits -# -# with_dtests_jobs: &with_dtest_jobs -# jobs: -# - build -# # Java 8 unit tests will be run automatically -# - j8_unit_tests: -# requires: -# - build -# - j8_jvm_dtests: -# requires: -# - build -# # specialized unit tests (all run on request using Java 8) -# - start_utests_long: -# type: approval -# requires: -# - build -# - utests_long: -# requires: -# - start_utests_long -# - start_utests_compression: -# type: approval -# requires: -# - build -# - utests_compression: -# requires: -# - start_utests_compression -# # Java 8 dtests (on request) -# - start_j8_dtests: -# type: approval -# requires: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - start_j8_dtests -# - j8_dtests-no-vnodes: -# requires: -# - start_j8_dtests -# # Java 8 upgrade tests -# - start_upgrade_tests: -# type: approval -# requires: -# - build -# - j8_upgradetests-no-vnodes: -# requires: -# - start_upgrade_tests -# -# with_dtest_jobs_only: &with_dtest_jobs_only -# jobs: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - build -# - j8_dtests-no-vnodes: -# requires: -# - build -# -# workflows: -# version: 2 -# build_and_run_tests: *with_dtest_jobs -# #build_and_run_tests: *with_dtest_jobs_only -# -# executors: -# java8-executor: -# parameters: -# exec_resource_class: -# type: string -# default: medium -# docker: -# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 -# resource_class: << parameters.exec_resource_class >> -# working_directory: ~/ -# shell: /bin/bash -eo pipefail -l -# environment: -# <<: *default_env_vars -# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# -# jobs: -# build: -# executor: java8-executor -# parallelism: 1 # This job doesn't benefit from parallelism -# steps: -# - log_environment -# - clone_cassandra -# - build_cassandra -# - run_eclipse_warnings -# - persist_to_workspace: -# root: /home/cassandra -# paths: -# - cassandra -# - .m2 -# -# j8_unit_tests: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests -# -# j8_jvm_dtests: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: test-jvm-dtest-forking -# -# utests_long: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: long-test -# -# utests_compression: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests: -# target: testclasslist-compression -# -# j8_dtests-with-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_with_vnodes -# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_with_vnodes -# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' -# -# j8_dtests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_without_vnodes -# run_dtests_extra_args: '--skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_without_vnodes -# pytest_extra_args: '--skip-resource-intensive-tests' -# -# j8_upgradetests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_upgradetests_without_vnodes -# run_dtests_extra_args: '--execute-upgrade-tests' -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# tests_filter_pattern: '^upgrade_tests' -# - run_dtests: -# file_tag: j8_upgradetests_without_vnodes -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# pytest_extra_args: '--execute-upgrade-tests' -# -# commands: -# log_environment: -# steps: -# - run: -# name: Log Environment Information -# command: | -# echo '*** id ***' -# id -# echo '*** cat /proc/cpuinfo ***' -# cat /proc/cpuinfo -# echo '*** free -m ***' -# free -m -# echo '*** df -m ***' -# df -m -# echo '*** ifconfig -a ***' -# ifconfig -a -# echo '*** uname -a ***' -# uname -a -# echo '*** mount ***' -# mount -# echo '*** env ***' -# env -# echo '*** java ***' -# which java -# java -version -# -# clone_cassandra: -# steps: -# - run: -# name: Clone Cassandra Repository (via git) -# command: | -# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra -# -# clone_dtest: -# steps: -# - run: -# name: Clone Cassandra dtest Repository (via git) -# command: | -# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest -# -# build_cassandra: -# steps: -# - run: -# name: Build Cassandra -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. -# for x in $(seq 1 3); do -# ${ANT_HOME}/bin/ant clean jar -# RETURN=\"$?\" -# if [ \"${RETURN}\" -eq \"0\" ]; then -# break -# fi -# done -# # Exit, if we didn't build successfully -# if [ \"${RETURN}\" -ne \"0\" ]; then -# echo \"Build failed with exit code: ${RETURN}\" -# exit ${RETURN} -# fi -# no_output_timeout: 15m -# -# run_eclipse_warnings: -# steps: -# - run: -# name: Run eclipse-warnings -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# ant eclipse-warnings -# -# create_junit_containers: -# steps: -# - run: -# name: Determine Unit Tests to Run -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# rm -fr ~/cassandra-dtest/upgrade_tests -# echo \"***java tests***\" -# -# # get all of our unit test filenames -# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt -# -# # split up the unit tests into groups based on the number of containers we have -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt -# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\" -# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# -# no_output_timeout: 15m -# -# run_junit_tests: -# parameters: -# target: -# type: string -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant clean <> -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# run_parallel_junit_tests: -# parameters: -# target: -# type: string -# default: testclasslist -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# create_venv: -# steps: -# - run: -# name: Configure virtualenv and python Dependencies -# command: | -# # note, this should be super quick as all dependencies should be pre-installed in the docker image -# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated -# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and -# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt -# pip3 freeze -# -# create_dtest_containers: -# parameters: -# file_tag: -# type: string -# run_dtests_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# tests_filter_pattern: -# type: string -# default: '' -# steps: -# - run: -# name: Determine Tests to Run (<>) -# no_output_timeout: 5m -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# cd cassandra-dtest -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# -# if [ -n '<>' ]; then -# export <> -# fi -# -# echo \"***Collected DTests (<>)***\" -# set -eo pipefail && ./run_dtests.py <> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<>_raw --cassandra-dir=../cassandra -# if [ -z '<>' ]; then -# mv /tmp/all_dtest_tests_<>_raw /tmp/all_dtest_tests_<> -# else -# grep -e '<>' /tmp/all_dtest_tests_<>_raw > /tmp/all_dtest_tests_<> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; } -# fi -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<> > /tmp/split_dtest_tests_<>.txt -# cat /tmp/split_dtest_tests_<>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<>_final.txt -# cat /tmp/split_dtest_tests_<>_final.txt -# -# run_dtests: -# parameters: -# file_tag: -# type: string -# pytest_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# steps: -# - run: -# name: Run dtests (<>) -# no_output_timeout: 15m -# command: | -# echo \"cat /tmp/split_dtest_tests_<>_final.txt\" -# cat /tmp/split_dtest_tests_<>_final.txt -# -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# if [ -n '<>' ]; then -# export <> -# fi -# -# java -version -# cd ~/cassandra-dtest -# mkdir -p /tmp/dtest -# -# echo \"env: $(env)\" -# echo \"** done env\" -# mkdir -p /tmp/results/dtests -# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee -# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<>_final.txt` -# set -o pipefail && cd ~/cassandra-dtest && pytest <> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt -# - store_test_results: -# path: /tmp/results -# - store_artifacts: -# path: /tmp/dtest -# destination: dtest_<> -# - store_artifacts: -# path: ~/cassandra-dtest/logs -# destination: dtest_<>_logs \ No newline at end of file diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES index 0d56e23c84..d80b78d8de 100644 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@ -1,5 +1,51 @@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean test-jvm-upgrade-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -81,343 +127,6 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_unit_tests: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine Unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_jvm_dtests: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Run Unit Tests (test-jvm-dtest-forking) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant clean test-jvm-dtest-forking - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_long: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 1 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Run Unit Tests (long-test) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant clean long-test - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - utests_compression: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Determine Unit Tests to Run - command: | - # reminder: this code (along with all the steps) is independently executed on every circle container - # so the goal here is to get the circleci script to return the tests *this* container will run - # which we do via the `circleci` cli tool. - - rm -fr ~/cassandra-dtest/upgrade_tests - echo "***java tests***" - - # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt - - # split up the unit tests into groups based on the number of containers we have - set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" - cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt - no_output_timeout: 15m - - run: - name: Log Environment Information - command: | - echo '*** id ***' - id - echo '*** cat /proc/cpuinfo ***' - cat /proc/cpuinfo - echo '*** free -m ***' - free -m - echo '*** df -m ***' - df -m - echo '*** ifconfig -a ***' - ifconfig -a - echo '*** uname -a ***' - uname -a - echo '*** mount ***' - mount - echo '*** env ***' - env - echo '*** java ***' - which java - java -version - - run: - name: Run Unit Tests (testclasslist-compression) - command: | - export PATH=$JAVA_HOME/bin:$PATH - time mv ~/cassandra /tmp - cd /tmp/cassandra - ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - no_output_timeout: 15m - - store_test_results: - path: /tmp/cassandra/build/test/output/ - - store_artifacts: - path: /tmp/cassandra/build/test/output - destination: junitxml - - store_artifacts: - path: /tmp/cassandra/build/test/logs - destination: logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - j8_dtests-with-vnodes: - docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium - working_directory: ~/ - shell: /bin/bash -eo pipefail -l - parallelism: 4 - steps: - - attach_workspace: - at: /home/cassandra - - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies - command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 freeze - - run: - name: Determine Tests to Run (j8_with_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - - run: - name: Run dtests (j8_with_vnodes) - no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\ - \ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_with_vnodes - - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_with_vnodes_logs - environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - ANT_HOME: /usr/share/ant - - LANG: en_US.UTF-8 - - KEEP_TEST_DIR: true - - DEFAULT_DIR: /home/cassandra/cassandra-dtest - - PYTHONIOENCODING: utf-8 - - PYTHONUNBUFFERED: true - - CASS_DRIVER_NO_EXTENSIONS: true - - CASS_DRIVER_NO_CYTHON: true - - CASSANDRA_SKIP_SYNC: true - - DTEST_REPO: git://github.com/apache/cassandra-dtest.git - - DTEST_BRANCH: master - - CCM_MAX_HEAP_SIZE: 1024M - - CCM_HEAP_NEWSIZE: 256M - - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j8_dtests-no-vnodes: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -446,37 +155,11 @@ jobs: - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\ - \ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" + command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" - store_test_results: path: /tmp/results - store_artifacts: @@ -601,11 +284,408 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_unit_tests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine Unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_dtests-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n" + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_jvm_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean test-jvm-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_long: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (long-test) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant clean long-test + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_compression: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine Unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist-compression) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + dtest_jars_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 build_and_run_tests: jobs: - build + - start_jvm_upgrade_dtest: + type: approval + requires: + - build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build - j8_unit_tests: requires: - build @@ -643,453 +723,3 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests - -# Original config.yml file: -# version: 2.1 -# -# default_env_vars: &default_env_vars -# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# ANT_HOME: /usr/share/ant -# LANG: en_US.UTF-8 -# KEEP_TEST_DIR: true -# DEFAULT_DIR: /home/cassandra/cassandra-dtest -# PYTHONIOENCODING: utf-8 -# PYTHONUNBUFFERED: true -# CASS_DRIVER_NO_EXTENSIONS: true -# CASS_DRIVER_NO_CYTHON: true -# #Skip all syncing to disk to avoid performance issues in flaky CI environments -# CASSANDRA_SKIP_SYNC: true -# DTEST_REPO: git://github.com/apache/cassandra-dtest.git -# DTEST_BRANCH: master -# CCM_MAX_HEAP_SIZE: 1024M -# CCM_HEAP_NEWSIZE: 256M -# -# j8_par_executor: &j8_par_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 4 -# -# j8_seq_executor: &j8_seq_executor -# executor: -# name: java8-executor -# #exec_resource_class: xlarge -# parallelism: 1 # sequential, single container tests: no parallelism benefits -# -# with_dtests_jobs: &with_dtest_jobs -# jobs: -# - build -# # Java 8 unit tests will be run automatically -# - j8_unit_tests: -# requires: -# - build -# - j8_jvm_dtests: -# requires: -# - build -# # specialized unit tests (all run on request using Java 8) -# - start_utests_long: -# type: approval -# requires: -# - build -# - utests_long: -# requires: -# - start_utests_long -# - start_utests_compression: -# type: approval -# requires: -# - build -# - utests_compression: -# requires: -# - start_utests_compression -# # Java 8 dtests (on request) -# - start_j8_dtests: -# type: approval -# requires: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - start_j8_dtests -# - j8_dtests-no-vnodes: -# requires: -# - start_j8_dtests -# # Java 8 upgrade tests -# - start_upgrade_tests: -# type: approval -# requires: -# - build -# - j8_upgradetests-no-vnodes: -# requires: -# - start_upgrade_tests -# -# with_dtest_jobs_only: &with_dtest_jobs_only -# jobs: -# - build -# - j8_dtests-with-vnodes: -# requires: -# - build -# - j8_dtests-no-vnodes: -# requires: -# - build -# -# workflows: -# version: 2 -# build_and_run_tests: *with_dtest_jobs -# #build_and_run_tests: *with_dtest_jobs_only -# -# executors: -# java8-executor: -# parameters: -# exec_resource_class: -# type: string -# default: medium -# docker: -# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 -# resource_class: << parameters.exec_resource_class >> -# working_directory: ~/ -# shell: /bin/bash -eo pipefail -l -# environment: -# <<: *default_env_vars -# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 -# -# jobs: -# build: -# executor: java8-executor -# parallelism: 1 # This job doesn't benefit from parallelism -# steps: -# - log_environment -# - clone_cassandra -# - build_cassandra -# - run_eclipse_warnings -# - persist_to_workspace: -# root: /home/cassandra -# paths: -# - cassandra -# - .m2 -# -# j8_unit_tests: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests -# -# j8_jvm_dtests: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: test-jvm-dtest-forking -# -# utests_long: -# <<: *j8_seq_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - run_junit_tests: -# target: long-test -# -# utests_compression: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - create_junit_containers -# - log_environment -# - run_parallel_junit_tests: -# target: testclasslist-compression -# -# j8_dtests-with-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_with_vnodes -# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_with_vnodes -# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' -# -# j8_dtests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_without_vnodes -# run_dtests_extra_args: '--skip-resource-intensive-tests' -# - run_dtests: -# file_tag: j8_without_vnodes -# pytest_extra_args: '--skip-resource-intensive-tests' -# -# j8_upgradetests-no-vnodes: -# <<: *j8_par_executor -# steps: -# - attach_workspace: -# at: /home/cassandra -# - clone_dtest -# - create_venv -# - create_dtest_containers: -# file_tag: j8_upgradetests_without_vnodes -# run_dtests_extra_args: '--execute-upgrade-tests' -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# tests_filter_pattern: '^upgrade_tests' -# - run_dtests: -# file_tag: j8_upgradetests_without_vnodes -# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' -# pytest_extra_args: '--execute-upgrade-tests' -# -# commands: -# log_environment: -# steps: -# - run: -# name: Log Environment Information -# command: | -# echo '*** id ***' -# id -# echo '*** cat /proc/cpuinfo ***' -# cat /proc/cpuinfo -# echo '*** free -m ***' -# free -m -# echo '*** df -m ***' -# df -m -# echo '*** ifconfig -a ***' -# ifconfig -a -# echo '*** uname -a ***' -# uname -a -# echo '*** mount ***' -# mount -# echo '*** env ***' -# env -# echo '*** java ***' -# which java -# java -version -# -# clone_cassandra: -# steps: -# - run: -# name: Clone Cassandra Repository (via git) -# command: | -# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra -# -# clone_dtest: -# steps: -# - run: -# name: Clone Cassandra dtest Repository (via git) -# command: | -# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest -# -# build_cassandra: -# steps: -# - run: -# name: Build Cassandra -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. -# for x in $(seq 1 3); do -# ${ANT_HOME}/bin/ant clean jar -# RETURN=\"$?\" -# if [ \"${RETURN}\" -eq \"0\" ]; then -# break -# fi -# done -# # Exit, if we didn't build successfully -# if [ \"${RETURN}\" -ne \"0\" ]; then -# echo \"Build failed with exit code: ${RETURN}\" -# exit ${RETURN} -# fi -# no_output_timeout: 15m -# -# run_eclipse_warnings: -# steps: -# - run: -# name: Run eclipse-warnings -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# cd ~/cassandra -# ant eclipse-warnings -# -# create_junit_containers: -# steps: -# - run: -# name: Determine Unit Tests to Run -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# rm -fr ~/cassandra-dtest/upgrade_tests -# echo \"***java tests***\" -# -# # get all of our unit test filenames -# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt -# -# # split up the unit tests into groups based on the number of containers we have -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt -# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\" -# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -# -# no_output_timeout: 15m -# -# run_junit_tests: -# parameters: -# target: -# type: string -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant clean <> -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# run_parallel_junit_tests: -# parameters: -# target: -# type: string -# default: testclasslist -# no_output_timeout: -# type: string -# default: 15m -# steps: -# - run: -# name: Run Unit Tests (<>) -# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case -# # based on Java 11 builds. -# command: | -# export PATH=$JAVA_HOME/bin:$PATH -# time mv ~/cassandra /tmp -# cd /tmp/cassandra -# ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -# no_output_timeout: <> -# - store_test_results: -# path: /tmp/cassandra/build/test/output/ -# - store_artifacts: -# path: /tmp/cassandra/build/test/output -# destination: junitxml -# - store_artifacts: -# path: /tmp/cassandra/build/test/logs -# destination: logs -# -# create_venv: -# steps: -# - run: -# name: Configure virtualenv and python Dependencies -# command: | -# # note, this should be super quick as all dependencies should be pre-installed in the docker image -# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated -# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and -# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt -# pip3 freeze -# -# create_dtest_containers: -# parameters: -# file_tag: -# type: string -# run_dtests_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# tests_filter_pattern: -# type: string -# default: '' -# steps: -# - run: -# name: Determine Tests to Run (<>) -# no_output_timeout: 5m -# command: | -# # reminder: this code (along with all the steps) is independently executed on every circle container -# # so the goal here is to get the circleci script to return the tests *this* container will run -# # which we do via the `circleci` cli tool. -# -# cd cassandra-dtest -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# -# if [ -n '<>' ]; then -# export <> -# fi -# -# echo \"***Collected DTests (<>)***\" -# set -eo pipefail && ./run_dtests.py <> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<>_raw --cassandra-dir=../cassandra -# if [ -z '<>' ]; then -# mv /tmp/all_dtest_tests_<>_raw /tmp/all_dtest_tests_<> -# else -# grep -e '<>' /tmp/all_dtest_tests_<>_raw > /tmp/all_dtest_tests_<> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; } -# fi -# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<> > /tmp/split_dtest_tests_<>.txt -# cat /tmp/split_dtest_tests_<>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<>_final.txt -# cat /tmp/split_dtest_tests_<>_final.txt -# -# run_dtests: -# parameters: -# file_tag: -# type: string -# pytest_extra_args: -# type: string -# default: '' -# extra_env_args: -# type: string -# default: '' -# steps: -# - run: -# name: Run dtests (<>) -# no_output_timeout: 15m -# command: | -# echo \"cat /tmp/split_dtest_tests_<>_final.txt\" -# cat /tmp/split_dtest_tests_<>_final.txt -# -# source ~/env/bin/activate -# export PATH=$JAVA_HOME/bin:$PATH -# if [ -n '<>' ]; then -# export <> -# fi -# -# java -version -# cd ~/cassandra-dtest -# mkdir -p /tmp/dtest -# -# echo \"env: $(env)\" -# echo \"** done env\" -# mkdir -p /tmp/results/dtests -# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee -# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<>_final.txt` -# set -o pipefail && cd ~/cassandra-dtest && pytest <> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt -# - store_test_results: -# path: /tmp/results -# - store_artifacts: -# path: /tmp/dtest -# destination: dtest_<> -# - store_artifacts: -# path: ~/cassandra-dtest/logs -# destination: dtest_<>_logs \ No newline at end of file diff --git a/CHANGES.txt b/CHANGES.txt index 2edc399d0a..025e2aec96 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0.20 + * Run in-jvm upgrade dtests in circleci (CASSANDRA-15506) * Include updates to static column in mutation size calculations (CASSANDRA-15293) * Fix point-in-time recoevery ignoring timestamp of updates to static columns (CASSANDRA-15292) * GC logs are also put under $CASSANDRA_LOG_DIR (CASSANDRA-14306) diff --git a/build.xml b/build.xml index 5d86a3f3db..d24f7456f8 100644 --- a/build.xml +++ b/build.xml @@ -66,6 +66,8 @@ + + @@ -1810,6 +1812,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java b/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java index a7ad400566..e5669109e1 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java +++ b/test/distributed/org/apache/cassandra/distributed/test/TestLocator.java @@ -39,14 +39,18 @@ public class TestLocator public static void main(String[] args) throws Throwable { String outputFileName = defaultOutputFileName; - if (args.length == 1) + if (args.length >= 1) { outputFileName = args[0]; } + String testPackage = TestLocator.testPackage; + if (args.length == 2) + testPackage = args[1]; try (FileWriter fileWriter = new FileWriter(outputFileName); PrintWriter printWriter = new PrintWriter(fileWriter)) { printWriter.println("#!/bin/bash"); + printWriter.println("ret=0"); for (Class testClass : locateClasses(testPackage)) { for (Method method : testClass.getMethods()) @@ -57,8 +61,10 @@ public class TestLocator printWriter.println(String.format(testCommandFormat, testClass.getName(), method.getName())); + printWriter.println("if [ $? -ne 0 ]; then ret=1; fi"); } } + printWriter.println("exit $ret"); } }