diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml index b55b1fb0ca..5cbe288fc4 100644 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@ -23,6 +23,18 @@ j8_par_executor: &j8_par_executor #exec_resource_class: xlarge parallelism: 4 +j8_small_par_executor: &j8_small_par_executor + executor: + name: java8-executor + #exec_resource_class: xlarge + parallelism: 1 + +j8_medium_par_executor: &j8_medium_par_executor + executor: + name: java8-executor + #exec_resource_class: xlarge + parallelism: 1 + j8_seq_executor: &j8_seq_executor executor: name: java8-executor @@ -63,14 +75,13 @@ with_dtests_jobs: &with_dtest_jobs - start_utests_stress - start_jvm_upgrade_dtest: type: approval + - j8_dtest_jars_build: requires: - build - - dtest_jars_build: - requires: - start_jvm_upgrade_dtest - j8_jvm_upgrade_dtests: requires: - - dtest_jars_build + - j8_dtest_jars_build # Java 8 dtests (on request) - start_j8_dtests: type: approval @@ -137,7 +148,7 @@ jobs: - cassandra - .m2 - dtest_jars_build: + j8_dtest_jars_build: executor: java8-executor parallelism: 1 steps: @@ -159,20 +170,28 @@ jobs: - run_parallel_junit_tests j8_jvm_dtests: - <<: *j8_seq_executor + <<: *j8_small_par_executor steps: - attach_workspace: at: /home/cassandra - - run_junit_tests: - target: test-jvm-dtest-forking + - create_junit_containers: + classlistprefix: distributed + extra_filters: "| grep -v upgrade" + - log_environment + - run_parallel_junit_tests: + classlistprefix: distributed j8_jvm_upgrade_dtests: - <<: *j8_seq_executor + <<: *j8_medium_par_executor steps: - attach_workspace: at: /home/cassandra - - run_junit_tests: - target: test-jvm-upgrade-dtest-forking + - create_junit_containers: + classlistprefix: distributed + extra_filters: "| grep upgrade" + - log_environment + - run_parallel_junit_tests: + classlistprefix: distributed utests_long: <<: *j8_seq_executor @@ -358,9 +377,16 @@ commands: ant eclipse-warnings create_junit_containers: + parameters: + classlistprefix: + type: string + default: unit + extra_filters: + type: string + default: "" steps: - run: - name: Determine Unit Tests to Run + name: Determine <> Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -370,11 +396,11 @@ commands: echo "***java tests***" # get all of our unit test filenames - set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/<>/**/*.java" > /tmp/all_java_unit_tests.txt # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/<>/;;g" | grep "Test\.java$" <> > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt @@ -387,6 +413,9 @@ commands: no_output_timeout: type: string default: 15m + classlistprefix: + type: string + default: unit steps: - run: name: Run Unit Tests (<>) @@ -399,7 +428,7 @@ commands: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean <> + ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<> no_output_timeout: <> - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -418,6 +447,9 @@ commands: no_output_timeout: type: string default: 15m + classlistprefix: + type: string + default: unit steps: - run: name: Run Unit Tests (<>) @@ -427,7 +459,10 @@ commands: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra - ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi + ant <> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<> no_output_timeout: <> - store_test_results: path: /tmp/cassandra/build/test/output/ diff --git a/.circleci/config-2_1.yml.high_res.patch b/.circleci/config-2_1.yml.high_res.patch index 22f434dd05..3c85668c75 100644 --- a/.circleci/config-2_1.yml.high_res.patch +++ b/.circleci/config-2_1.yml.high_res.patch @@ -10,7 +10,19 @@ --- > exec_resource_class: xlarge > parallelism: 100 -29c29 +29,30c29,30 +< #exec_resource_class: xlarge +< parallelism: 1 +--- +> exec_resource_class: xlarge +> parallelism: 2 +35,36c35,36 +< #exec_resource_class: xlarge +< parallelism: 1 +--- +> exec_resource_class: xlarge +> parallelism: 2 +41c41 < #exec_resource_class: xlarge --- > exec_resource_class: xlarge diff --git a/.circleci/config.yml b/.circleci/config.yml index e0c65f41ae..4108359788 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,48 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp @@ -19,7 +60,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean test-jvm-upgrade-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -303,7 +344,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean stress-test + ant stress-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -341,7 +382,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -355,7 +396,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -387,6 +428,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: @@ -483,7 +527,48 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (test-jvm-dtest-forking) + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp @@ -491,7 +576,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean test-jvm-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -537,7 +622,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean long-test + ant long-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -575,7 +660,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -589,7 +674,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -621,6 +706,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: @@ -648,7 +736,7 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - dtest_jars_build: + j8_dtest_jars_build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 resource_class: medium @@ -751,14 +839,13 @@ workflows: - start_utests_stress - start_jvm_upgrade_dtest: type: approval + - j8_dtest_jars_build: requires: - build - - dtest_jars_build: - requires: - start_jvm_upgrade_dtest - j8_jvm_upgrade_dtests: requires: - - dtest_jars_build + - j8_dtest_jars_build - start_j8_dtests: type: approval requires: diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES index c6688d9d9f..ac61b7f931 100644 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@ -6,12 +6,53 @@ jobs: resource_class: xlarge working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 1 + parallelism: 2 steps: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp @@ -19,7 +60,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean test-jvm-upgrade-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -303,7 +344,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean stress-test + ant stress-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -341,7 +382,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -355,7 +396,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -387,6 +428,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: @@ -478,12 +522,53 @@ jobs: resource_class: xlarge working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 1 + parallelism: 2 steps: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (test-jvm-dtest-forking) + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp @@ -491,7 +576,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean test-jvm-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -537,7 +622,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean long-test + ant long-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -575,7 +660,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -589,7 +674,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -621,6 +706,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: @@ -648,7 +736,7 @@ jobs: - CCM_HEAP_NEWSIZE: 512M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - dtest_jars_build: + j8_dtest_jars_build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 resource_class: medium @@ -751,14 +839,13 @@ workflows: - start_utests_stress - start_jvm_upgrade_dtest: type: approval + - j8_dtest_jars_build: requires: - build - - dtest_jars_build: - requires: - start_jvm_upgrade_dtest - j8_jvm_upgrade_dtests: requires: - - dtest_jars_build + - j8_dtest_jars_build - start_j8_dtests: type: approval requires: diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES index e0c65f41ae..4108359788 100644 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@ -11,7 +11,48 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp @@ -19,7 +60,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean test-jvm-upgrade-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -303,7 +344,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean stress-test + ant stress-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -341,7 +382,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -355,7 +396,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -387,6 +428,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: @@ -483,7 +527,48 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Run Unit Tests (test-jvm-dtest-forking) + name: Determine distributed Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp @@ -491,7 +576,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean test-jvm-dtest-forking + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -537,7 +622,7 @@ jobs: if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant clean long-test + ant long-test -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@ -575,7 +660,7 @@ jobs: - attach_workspace: at: /home/cassandra - run: - name: Determine Unit Tests to Run + name: Determine unit Tests to Run command: | # reminder: this code (along with all the steps) is independently executed on every circle container # so the goal here is to get the circleci script to return the tests *this* container will run @@ -589,7 +674,7 @@ jobs: # split up the unit tests into groups based on the number of containers we have set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt - set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt no_output_timeout: 15m @@ -621,6 +706,9 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi ant testclasslist-compression -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m - store_test_results: @@ -648,7 +736,7 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - dtest_jars_build: + j8_dtest_jars_build: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 resource_class: medium @@ -751,14 +839,13 @@ workflows: - start_utests_stress - start_jvm_upgrade_dtest: type: approval + - j8_dtest_jars_build: requires: - build - - dtest_jars_build: - requires: - start_jvm_upgrade_dtest - j8_jvm_upgrade_dtests: requires: - - dtest_jars_build + - j8_dtest_jars_build - start_j8_dtests: type: approval requires: